In [1]:
import logging, os
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
import warnings
warnings.filterwarnings('ignore')
In [2]:
import pickle as pkl
import pandas as pd
import numpy as np
from numpy.random import RandomState
rng = RandomState(93748573)
from pprint import pprint
In [3]:
from gensim import models
from gensim.corpora import Dictionary
# Use coherence model to measure the LDA models generated
from gensim.models.coherencemodel import CoherenceModel
2018-03-28 18:44:18,401 : INFO : 'pattern' package not found; tag filters are not available for English
In [4]:
import matplotlib.pyplot as plt
plt.style.use(['seaborn-poster'])
%matplotlib inline
import seaborn as sns
from wordcloud import WordCloud
import pyLDAvis.gensim as gensimvis
import pyLDAvis
pyLDAvis.enable_notebook()
from IPython.display import HTML
Topic Modelling on data_schoolofinf¶
! Because of the need to run this script for long hours to find the "best" k number of topics, we use scripts located in src for each model
In [5]:
DATA_DIR = '../../data/data_schoolofinf/'
Corpus used for topic modelling¶
In [6]:
# Import the dataset:
df_combined_toks = pd.read_pickle(os.path.join(DATA_DIR,'toks', 'toks.combined.pkl'))
df_combined_toks.head(3)
Out[6]:
| year | toks_metada | toks_pdf2txt | |
|---|---|---|---|
| pub_id | |||
| 400818dc-63af-4a26-80c5-906f98e1f8ab | 1989 | [ballooning, stability, analysis, jet, hmode, ... | |
| 18b1a861-afef-4fff-bc80-d02e05be18c4 | 2013 | [query, processing, data, integration, chapter... | |
| 309fdbfc-227b-4588-9264-f0f4e3cadfcb | 1994 | [comprehension, syntax, syntax, comprehension,... | [comprehension, syntax, citation, published, v... |
In [7]:
# Remove unrelevant documents
df_combined_toks = df_combined_toks.drop(
df_combined_toks[(df_combined_toks.year < 1997) | (df_combined_toks.year > 2017)].index)
# Add empty list so that concat_toks is just an addition of the two list.
df_combined_toks[
'toks_pdf2txt'] = df_combined_toks.toks_pdf2txt.apply(
lambda x: [] if not len(x) else x)
df_combined_toks[
'toks_metada'] = df_combined_toks.toks_metada.apply(
lambda x: [] if not len(x) else x)
In [11]:
def get_score(num_topic, dictionary, corpus, texts, coherence, model_dir):
"""
Load a particular topic model and evaluate it with a given measure.
"""
_model = models.LdaModel.load('{}/ldamodel_nb_topics_{}'.format(
model_dir, int(num_topic)))
# topn was 20 by default in the original function
toptopics = _model.top_topics(
corpus=corpus, texts=texts, coherence=coherence, dictionary=dictionary, topn=15)
avg = sum(t[1] for t in toptopics) / num_topic # Return the average measure
return avg
In [9]:
def plot_results(df, fname):
fig = plt.figure(figsize=(9, 9))
ax = fig.add_subplot(111)
ax = df.plot(x='num_topic', y='c_v', label='c_v score', legend=True, ax=ax)
best = df.c_v.argmax()
ax.scatter(x=df.num_topic.iloc[best], y=df.iloc[best].c_v, c='g', marker='*', s=150)
ax.set_ylabel('c_v score')
ax = df.plot(
x='num_topic',
y='u_mass',
secondary_y=True,
label='u_mass score',
legend=True,
ax=ax)
ax.set_ylabel('u_mass score')
fig.savefig('IMG/{}.png'.format(fname), format='png', bbox_inches='tight')
return ax
1997-2017 Publications¶
tmfull : metadata + pdf¶
In [44]:
dict_tmfull = Dictionary.load(os.path.join(DATA_DIR, 'corpora', 'dictionary.all'))
df_tmfull = df_combined_toks # create a copy
df_tmfull['concat_toks'] = df_tmfull.apply(
lambda row: row.toks_metada + row.toks_pdf2txt, axis=1)
# Create a bow tagging for each publication:
df_tmfull['bow'] = df_tmfull['toks_metada'].apply(dict_tmfull.doc2bow)
# Generate a corpus based on the tokens, which we will be using later
corpus_tmfull = df_tmfull.bow.tolist()
text_tmfull = df_tmfull.concat_toks.tolist()
# df_all = df_combined_toks.copy()
# df_all['concat_toks'] = df_combined_toks.apply(
# lambda row: row.toks_metada + row.toks_pdf2txt, axis=1)
# # Create a bow tagging for each publication:
# df_all['bow'] = df_all['concat_toks'].apply(dict_all.doc2bow)
# # Generate a corpus based on the tokens, which we will be using later
# corpus_all = df_all.bow.tolist()
2018-03-28 00:33:27,057 : INFO : loading Dictionary object from ../../data/data_schoolofinf/corpora/dictionary.all 2018-03-28 00:33:27,156 : INFO : loaded ../../data/data_schoolofinf/corpora/dictionary.all
In [27]:
df_tmfull.head(3)
Out[27]:
| year | toks_metada | toks_pdf2txt | concat_toks | bow | |
|---|---|---|---|---|---|
| pub_id | |||||
| 18b1a861-afef-4fff-bc80-d02e05be18c4 | 2013 | [query, processing, data, integration, chapter... | [] | [query, processing, data, integration, chapter... | [(0, 8), (1, 4), (2, 4), (3, 4), (4, 1), (5, 1... |
| d5814bab-5fc2-4c31-92b7-543c7ce75cb4 | 2012 | [evaluation, speaker, verification, security, ... | [evaluation, speaker, verification, security, ... | [evaluation, speaker, verification, security, ... | [(1, 1), (30, 1), (65, 1), (66, 4), (67, 2), (... |
| 880944d3-26db-4003-9186-130bf3202941 | 2014 | [openairinterface, flexible, platform, researc... | [] | [openairinterface, flexible, platform, researc... | [(24, 1), (65, 1), (83, 1), (85, 1), (120, 1),... |
In [12]:
tmfull_score = pd.read_csv('../src/tmfull/scores.csv')
Evaluation of model¶
In [22]:
tmfull_score['c_v'] = tmfull_score.apply(lambda row: get_score(row.num_topic, dict_tmfull, texts=text_tmfull, corpus=corpus_tmfull,
coherence='c_v', model_dir='../src/tmfull'),
axis=1)
2018-03-26 13:47:35,786 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_10 2018-03-26 13:47:35,792 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-26 13:47:35,796 : INFO : setting ignored attribute state to None 2018-03-26 13:47:35,797 : INFO : setting ignored attribute dispatcher to None 2018-03-26 13:47:35,797 : INFO : setting ignored attribute id2word to None 2018-03-26 13:47:35,798 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_10 2018-03-26 13:47:35,799 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_10.state 2018-03-26 13:47:35,836 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_10.state 2018-03-26 13:47:35,875 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 13:47:35,987 : INFO : 1 batches submitted to accumulate stats from 64 documents (199345 virtual) 2018-03-26 13:47:36,050 : INFO : 2 batches submitted to accumulate stats from 128 documents (369516 virtual) 2018-03-26 13:47:36,118 : INFO : 3 batches submitted to accumulate stats from 192 documents (512545 virtual) 2018-03-26 13:47:36,318 : INFO : 4 batches submitted to accumulate stats from 256 documents (814548 virtual) 2018-03-26 13:47:36,447 : INFO : 5 batches submitted to accumulate stats from 320 documents (993019 virtual) 2018-03-26 13:47:36,516 : INFO : 6 batches submitted to accumulate stats from 384 documents (1115499 virtual) 2018-03-26 13:47:40,708 : INFO : 7 batches submitted to accumulate stats from 448 documents (1282915 virtual) 2018-03-26 13:47:41,638 : INFO : 8 batches submitted to accumulate stats from 512 documents (1473567 virtual) 2018-03-26 13:47:42,547 : INFO : 9 batches submitted to accumulate stats from 576 documents (1651693 virtual) 2018-03-26 13:47:46,586 : INFO : 10 batches submitted to accumulate stats from 640 documents (1780925 virtual) 2018-03-26 13:47:47,431 : INFO : 11 batches submitted to accumulate stats from 704 documents (1944082 virtual) 2018-03-26 13:47:51,351 : INFO : 12 batches submitted to accumulate stats from 768 documents (2136313 virtual) 2018-03-26 13:47:51,894 : INFO : 13 batches submitted to accumulate stats from 832 documents (2288903 virtual) 2018-03-26 13:47:53,772 : INFO : 14 batches submitted to accumulate stats from 896 documents (2386402 virtual) 2018-03-26 13:47:56,258 : INFO : 15 batches submitted to accumulate stats from 960 documents (2627818 virtual) 2018-03-26 13:47:57,210 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2906274 virtual) 2018-03-26 13:47:59,211 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3101091 virtual) 2018-03-26 13:48:01,900 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3283982 virtual) 2018-03-26 13:48:02,381 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3516671 virtual) 2018-03-26 13:48:02,666 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3702552 virtual) 2018-03-26 13:48:09,094 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3835632 virtual) 2018-03-26 13:48:09,630 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4022982 virtual) 2018-03-26 13:48:11,572 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4822121 virtual) 2018-03-26 13:48:14,983 : INFO : 24 batches submitted to accumulate stats from 1536 documents (4957976 virtual) 2018-03-26 13:48:17,367 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5193532 virtual) 2018-03-26 13:48:17,492 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5337245 virtual) 2018-03-26 13:48:19,367 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5536112 virtual) 2018-03-26 13:48:23,511 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5688049 virtual) 2018-03-26 13:48:23,686 : INFO : 29 batches submitted to accumulate stats from 1856 documents (5985845 virtual) 2018-03-26 13:48:28,048 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6124898 virtual) 2018-03-26 13:48:31,014 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6248278 virtual) 2018-03-26 13:48:34,914 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7869468 virtual) 2018-03-26 13:48:36,347 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8318451 virtual) 2018-03-26 13:48:40,862 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8552262 virtual) 2018-03-26 13:48:43,820 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8693752 virtual) 2018-03-26 13:48:44,412 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8835184 virtual) 2018-03-26 13:48:44,565 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9010431 virtual) 2018-03-26 13:48:51,895 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9152495 virtual) 2018-03-26 13:48:56,351 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9319129 virtual) 2018-03-26 13:48:57,204 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9510589 virtual) 2018-03-26 13:49:00,971 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9644240 virtual) 2018-03-26 13:49:02,660 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9847554 virtual) 2018-03-26 13:49:05,422 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9997705 virtual) 2018-03-26 13:49:08,196 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10179956 virtual) 2018-03-26 13:49:11,521 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10332144 virtual) 2018-03-26 13:49:12,634 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10456735 virtual) 2018-03-26 13:49:17,320 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10639685 virtual) 2018-03-26 13:49:18,040 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10801759 virtual) 2018-03-26 13:49:22,851 : INFO : 49 batches submitted to accumulate stats from 3136 documents (10996966 virtual) 2018-03-26 13:49:23,047 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11156827 virtual) 2018-03-26 13:49:27,060 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11287237 virtual) 2018-03-26 13:49:28,716 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11439421 virtual) 2018-03-26 13:49:32,385 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11624429 virtual) 2018-03-26 13:49:35,078 : INFO : 54 batches submitted to accumulate stats from 3456 documents (11747573 virtual) 2018-03-26 13:49:36,379 : INFO : 55 batches submitted to accumulate stats from 3520 documents (11930107 virtual) 2018-03-26 13:49:37,631 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12084512 virtual) 2018-03-26 13:49:39,399 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12259333 virtual) 2018-03-26 13:49:41,136 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12397710 virtual) 2018-03-26 13:49:43,418 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12567336 virtual) 2018-03-26 13:49:43,875 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12788549 virtual) 2018-03-26 13:49:47,547 : INFO : 61 batches submitted to accumulate stats from 3904 documents (12921508 virtual) 2018-03-26 13:49:48,777 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13049829 virtual) 2018-03-26 13:49:49,843 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13192119 virtual) 2018-03-26 13:49:52,049 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13327289 virtual) 2018-03-26 13:49:54,373 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13470373 virtual) 2018-03-26 13:49:56,349 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13635176 virtual) 2018-03-26 13:49:57,064 : INFO : 67 batches submitted to accumulate stats from 4288 documents (13805955 virtual) 2018-03-26 13:49:58,512 : INFO : 68 batches submitted to accumulate stats from 4352 documents (13969906 virtual) 2018-03-26 13:50:00,864 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14092170 virtual) 2018-03-26 13:50:01,481 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14337998 virtual) 2018-03-26 13:50:03,293 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14553617 virtual) 2018-03-26 13:50:06,472 : INFO : 72 batches submitted to accumulate stats from 4608 documents (14704884 virtual) 2018-03-26 13:50:06,905 : INFO : 73 batches submitted to accumulate stats from 4672 documents (14871883 virtual) 2018-03-26 13:50:08,650 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15054501 virtual) 2018-03-26 13:50:10,471 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15208896 virtual) 2018-03-26 13:50:14,446 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15343110 virtual) 2018-03-26 13:50:15,126 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15494159 virtual) 2018-03-26 13:50:15,763 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15698871 virtual) 2018-03-26 13:50:19,858 : INFO : 79 batches submitted to accumulate stats from 5056 documents (15817169 virtual) 2018-03-26 13:50:20,774 : INFO : 80 batches submitted to accumulate stats from 5120 documents (15985897 virtual) 2018-03-26 13:50:21,318 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16142478 virtual) 2018-03-26 13:50:24,387 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16344928 virtual) 2018-03-26 13:50:25,774 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16470829 virtual) 2018-03-26 13:50:27,841 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16757121 virtual) 2018-03-26 13:50:28,321 : INFO : 85 batches submitted to accumulate stats from 5440 documents (16914212 virtual) 2018-03-26 13:50:31,193 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17156735 virtual) 2018-03-26 13:50:32,799 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17307277 virtual) 2018-03-26 13:50:34,961 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17442636 virtual) 2018-03-26 13:50:35,168 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17603214 virtual) 2018-03-26 13:50:40,140 : INFO : 90 batches submitted to accumulate stats from 5760 documents (17779589 virtual) 2018-03-26 13:50:41,439 : INFO : 91 batches submitted to accumulate stats from 5824 documents (17949276 virtual) 2018-03-26 13:50:42,645 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18222062 virtual) 2018-03-26 13:50:44,979 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18369331 virtual) 2018-03-26 13:50:46,005 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18501201 virtual) 2018-03-26 13:50:47,854 : INFO : 95 batches submitted to accumulate stats from 6080 documents (18666255 virtual) 2018-03-26 13:50:50,791 : INFO : 96 batches submitted to accumulate stats from 6144 documents (18794078 virtual) 2018-03-26 13:50:51,677 : INFO : 97 batches submitted to accumulate stats from 6208 documents (18917249 virtual) 2018-03-26 13:50:55,728 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19114142 virtual) 2018-03-26 13:50:55,909 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19305437 virtual) 2018-03-26 13:50:56,770 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19537663 virtual) 2018-03-26 13:51:00,301 : INFO : 101 batches submitted to accumulate stats from 6464 documents (19732331 virtual) 2018-03-26 13:51:00,814 : INFO : 102 batches submitted to accumulate stats from 6528 documents (19870955 virtual) 2018-03-26 13:51:01,050 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20040076 virtual) 2018-03-26 13:51:06,550 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20176279 virtual) 2018-03-26 13:51:07,474 : INFO : 105 batches submitted to accumulate stats from 6720 documents (21757558 virtual) 2018-03-26 13:51:08,607 : INFO : 106 batches submitted to accumulate stats from 6784 documents (21907357 virtual) 2018-03-26 13:51:11,946 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22069249 virtual) 2018-03-26 13:51:13,240 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22214543 virtual) 2018-03-26 13:51:14,222 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22363141 virtual) 2018-03-26 13:51:16,462 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22562613 virtual) 2018-03-26 13:51:18,918 : INFO : 111 batches submitted to accumulate stats from 7104 documents (22878625 virtual) 2018-03-26 13:51:21,848 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23021508 virtual) 2018-03-26 13:51:23,619 : INFO : 113 batches submitted to accumulate stats from 7232 documents (23306949 virtual) 2018-03-26 13:51:26,599 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23472186 virtual) 2018-03-26 13:51:29,334 : INFO : 115 batches submitted to accumulate stats from 7360 documents (23657107 virtual) 2018-03-26 13:51:33,658 : INFO : 116 batches submitted to accumulate stats from 7424 documents (23839423 virtual) 2018-03-26 13:51:36,771 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24012303 virtual) 2018-03-26 13:51:42,250 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24209443 virtual) 2018-03-26 13:51:42,524 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24289129 virtual) 2018-03-26 13:51:48,658 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24433726 virtual) 2018-03-26 13:51:48,761 : INFO : 121 batches submitted to accumulate stats from 7744 documents (24599421 virtual) 2018-03-26 13:51:54,173 : INFO : 122 batches submitted to accumulate stats from 7808 documents (24767467 virtual) 2018-03-26 13:51:55,058 : INFO : 123 batches submitted to accumulate stats from 7872 documents (24768770 virtual) 2018-03-26 13:52:02,246 : INFO : serializing accumulator to return to master... 2018-03-26 13:52:02,252 : INFO : accumulator serialized 2018-03-26 13:52:04,214 : INFO : serializing accumulator to return to master... 2018-03-26 13:52:04,287 : INFO : serializing accumulator to return to master... 2018-03-26 13:52:04,220 : INFO : accumulator serialized 2018-03-26 13:52:04,293 : INFO : accumulator serialized 2018-03-26 13:52:04,440 : INFO : 3 accumulators retrieved from output queue 2018-03-26 13:52:04,502 : INFO : accumulated word occurrence stats for 24866110 virtual documents 2018-03-26 13:52:04,889 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_15 2018-03-26 13:52:04,942 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_15.expElogbeta.npy with mmap=None 2018-03-26 13:52:05,012 : INFO : setting ignored attribute state to None 2018-03-26 13:52:05,013 : INFO : setting ignored attribute dispatcher to None 2018-03-26 13:52:05,014 : INFO : setting ignored attribute id2word to None 2018-03-26 13:52:05,014 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_15 2018-03-26 13:52:05,015 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_15.state 2018-03-26 13:52:05,157 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_15.state 2018-03-26 13:52:05,197 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 13:52:05,304 : INFO : 1 batches submitted to accumulate stats from 64 documents (190781 virtual) 2018-03-26 13:52:05,360 : INFO : 2 batches submitted to accumulate stats from 128 documents (359024 virtual) 2018-03-26 13:52:05,426 : INFO : 3 batches submitted to accumulate stats from 192 documents (501498 virtual) 2018-03-26 13:52:05,607 : INFO : 4 batches submitted to accumulate stats from 256 documents (805726 virtual) 2018-03-26 13:52:05,700 : INFO : 5 batches submitted to accumulate stats from 320 documents (961683 virtual) 2018-03-26 13:52:05,794 : INFO : 6 batches submitted to accumulate stats from 384 documents (1114916 virtual) 2018-03-26 13:52:10,842 : INFO : 7 batches submitted to accumulate stats from 448 documents (1276192 virtual) 2018-03-26 13:52:11,375 : INFO : 8 batches submitted to accumulate stats from 512 documents (1457760 virtual) 2018-03-26 13:52:12,555 : INFO : 9 batches submitted to accumulate stats from 576 documents (1638382 virtual) 2018-03-26 13:52:17,015 : INFO : 10 batches submitted to accumulate stats from 640 documents (1769108 virtual) 2018-03-26 13:52:18,191 : INFO : 11 batches submitted to accumulate stats from 704 documents (1937856 virtual) 2018-03-26 13:52:22,486 : INFO : 12 batches submitted to accumulate stats from 768 documents (2108009 virtual) 2018-03-26 13:52:22,766 : INFO : 13 batches submitted to accumulate stats from 832 documents (2283252 virtual) 2018-03-26 13:52:24,873 : INFO : 14 batches submitted to accumulate stats from 896 documents (2377051 virtual) 2018-03-26 13:52:27,535 : INFO : 15 batches submitted to accumulate stats from 960 documents (2557376 virtual) 2018-03-26 13:52:29,369 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2895611 virtual) 2018-03-26 13:52:31,463 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3078943 virtual) 2018-03-26 13:52:33,634 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3275641 virtual) 2018-03-26 13:52:34,948 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3487927 virtual) 2018-03-26 13:52:35,708 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3658504 virtual) 2018-03-26 13:52:40,296 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3817833 virtual) 2018-03-26 13:52:42,355 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4003012 virtual) 2018-03-26 13:52:47,426 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4810176 virtual) 2018-03-26 13:52:47,500 : INFO : 24 batches submitted to accumulate stats from 1536 documents (4939319 virtual) 2018-03-26 13:52:50,621 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5170714 virtual) 2018-03-26 13:52:53,288 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5306966 virtual) 2018-03-26 13:52:53,752 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5507377 virtual) 2018-03-26 13:52:57,406 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5658568 virtual) 2018-03-26 13:52:58,242 : INFO : 29 batches submitted to accumulate stats from 1856 documents (5897832 virtual) 2018-03-26 13:53:03,204 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6085636 virtual) 2018-03-26 13:53:05,578 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6218013 virtual) 2018-03-26 13:53:11,075 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7818318 virtual) 2018-03-26 13:53:11,300 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8269344 virtual) 2018-03-26 13:53:18,004 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8502008 virtual) 2018-03-26 13:53:19,571 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8649322 virtual) 2018-03-26 13:53:22,620 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8795366 virtual) 2018-03-26 13:53:23,217 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8940180 virtual) 2018-03-26 13:53:31,677 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9110548 virtual) 2018-03-26 13:53:36,819 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9274220 virtual) 2018-03-26 13:53:36,921 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9464846 virtual) 2018-03-26 13:53:41,986 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9607352 virtual) 2018-03-26 13:53:42,390 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9772638 virtual) 2018-03-26 13:53:48,890 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9945331 virtual) 2018-03-26 13:53:48,995 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10128456 virtual) 2018-03-26 13:53:54,492 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10290271 virtual) 2018-03-26 13:53:56,038 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10414163 virtual) 2018-03-26 13:54:00,372 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10573719 virtual) 2018-03-26 13:54:02,470 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10732313 virtual) 2018-03-26 13:54:06,823 : INFO : 49 batches submitted to accumulate stats from 3136 documents (10917163 virtual) 2018-03-26 13:54:08,188 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11105799 virtual) 2018-03-26 13:54:11,522 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11227662 virtual) 2018-03-26 13:54:13,900 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11338814 virtual) 2018-03-26 13:54:17,049 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11542715 virtual) 2018-03-26 13:54:18,206 : INFO : 54 batches submitted to accumulate stats from 3456 documents (11681618 virtual) 2018-03-26 13:54:20,725 : INFO : 55 batches submitted to accumulate stats from 3520 documents (11850313 virtual) 2018-03-26 13:54:22,865 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12006036 virtual) 2018-03-26 13:54:24,033 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12178093 virtual) 2018-03-26 13:54:24,620 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12349470 virtual) 2018-03-26 13:54:29,321 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12481952 virtual) 2018-03-26 13:54:30,550 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12671166 virtual) 2018-03-26 13:54:30,754 : INFO : 61 batches submitted to accumulate stats from 3904 documents (12841258 virtual) 2018-03-26 13:54:35,168 : INFO : 62 batches submitted to accumulate stats from 3968 documents (12966526 virtual) 2018-03-26 13:54:36,915 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13129771 virtual) 2018-03-26 13:54:36,995 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13251554 virtual) 2018-03-26 13:54:40,205 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13385528 virtual) 2018-03-26 13:54:43,214 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13514023 virtual) 2018-03-26 13:54:43,895 : INFO : 67 batches submitted to accumulate stats from 4288 documents (13700936 virtual) 2018-03-26 13:54:44,846 : INFO : 68 batches submitted to accumulate stats from 4352 documents (13876106 virtual) 2018-03-26 13:54:48,198 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14037796 virtual) 2018-03-26 13:54:49,264 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14232738 virtual) 2018-03-26 13:54:50,003 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14375075 virtual) 2018-03-26 13:54:53,087 : INFO : 72 batches submitted to accumulate stats from 4608 documents (14603623 virtual) 2018-03-26 13:54:56,122 : INFO : 73 batches submitted to accumulate stats from 4672 documents (14763512 virtual) 2018-03-26 13:54:56,376 : INFO : 74 batches submitted to accumulate stats from 4736 documents (14925369 virtual) 2018-03-26 13:54:59,272 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15099091 virtual) 2018-03-26 13:55:01,424 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15246786 virtual) 2018-03-26 13:55:02,991 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15392365 virtual) 2018-03-26 13:55:07,021 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15551092 virtual) 2018-03-26 13:55:07,583 : INFO : 79 batches submitted to accumulate stats from 5056 documents (15730314 virtual) 2018-03-26 13:55:09,073 : INFO : 80 batches submitted to accumulate stats from 5120 documents (15857682 virtual) 2018-03-26 13:55:13,017 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16013956 virtual) 2018-03-26 13:55:13,487 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16189905 virtual) 2018-03-26 13:55:14,706 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16372346 virtual) 2018-03-26 13:55:18,934 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16516794 virtual) 2018-03-26 13:55:19,497 : INFO : 85 batches submitted to accumulate stats from 5440 documents (16771733 virtual) 2018-03-26 13:55:20,194 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17016981 virtual) 2018-03-26 13:55:24,733 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17200316 virtual) 2018-03-26 13:55:26,012 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17335051 virtual) 2018-03-26 13:55:26,884 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17465926 virtual) 2018-03-26 13:55:30,032 : INFO : 90 batches submitted to accumulate stats from 5760 documents (17607357 virtual) 2018-03-26 13:55:34,421 : INFO : 91 batches submitted to accumulate stats from 5824 documents (17801681 virtual) 2018-03-26 13:55:35,637 : INFO : 92 batches submitted to accumulate stats from 5888 documents (17970785 virtual) 2018-03-26 13:55:36,730 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18260726 virtual) 2018-03-26 13:55:39,270 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18390668 virtual) 2018-03-26 13:55:40,537 : INFO : 95 batches submitted to accumulate stats from 6080 documents (18520024 virtual) 2018-03-26 13:55:42,116 : INFO : 96 batches submitted to accumulate stats from 6144 documents (18686825 virtual) 2018-03-26 13:55:46,837 : INFO : 97 batches submitted to accumulate stats from 6208 documents (18814091 virtual) 2018-03-26 13:55:46,903 : INFO : 98 batches submitted to accumulate stats from 6272 documents (18930020 virtual) 2018-03-26 13:55:51,751 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19159361 virtual) 2018-03-26 13:55:51,931 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19337468 virtual) 2018-03-26 13:55:53,382 : INFO : 101 batches submitted to accumulate stats from 6464 documents (19539953 virtual) 2018-03-26 13:55:56,707 : INFO : 102 batches submitted to accumulate stats from 6528 documents (19748403 virtual) 2018-03-26 13:55:57,779 : INFO : 103 batches submitted to accumulate stats from 6592 documents (19895948 virtual) 2018-03-26 13:55:57,996 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20050887 virtual) 2018-03-26 13:56:04,361 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20180427 virtual) 2018-03-26 13:56:05,247 : INFO : 106 batches submitted to accumulate stats from 6784 documents (21758448 virtual) 2018-03-26 13:56:05,606 : INFO : 107 batches submitted to accumulate stats from 6848 documents (21902338 virtual) 2018-03-26 13:56:10,834 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22073768 virtual) 2018-03-26 13:56:11,237 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22213171 virtual) 2018-03-26 13:56:12,118 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22357822 virtual) 2018-03-26 13:56:15,621 : INFO : 111 batches submitted to accumulate stats from 7104 documents (22557408 virtual) 2018-03-26 13:56:17,231 : INFO : 112 batches submitted to accumulate stats from 7168 documents (22883651 virtual) 2018-03-26 13:56:22,208 : INFO : 113 batches submitted to accumulate stats from 7232 documents (23016137 virtual) 2018-03-26 13:56:22,374 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23301488 virtual) 2018-03-26 13:56:27,584 : INFO : 115 batches submitted to accumulate stats from 7360 documents (23462969 virtual) 2018-03-26 13:56:28,726 : INFO : 116 batches submitted to accumulate stats from 7424 documents (23641537 virtual) 2018-03-26 13:56:33,368 : INFO : 117 batches submitted to accumulate stats from 7488 documents (23831867 virtual) 2018-03-26 13:56:39,751 : INFO : 118 batches submitted to accumulate stats from 7552 documents (23990691 virtual) 2018-03-26 13:56:43,150 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24186811 virtual) 2018-03-26 13:56:45,720 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24279798 virtual) 2018-03-26 13:56:49,462 : INFO : 121 batches submitted to accumulate stats from 7744 documents (24418944 virtual) 2018-03-26 13:56:52,615 : INFO : 122 batches submitted to accumulate stats from 7808 documents (24586634 virtual) 2018-03-26 13:56:55,202 : INFO : 123 batches submitted to accumulate stats from 7872 documents (24738432 virtual) 2018-03-26 13:56:58,602 : INFO : 124 batches submitted to accumulate stats from 7936 documents (24762775 virtual) 2018-03-26 13:57:06,744 : INFO : serializing accumulator to return to master... 2018-03-26 13:57:06,750 : INFO : accumulator serialized 2018-03-26 13:57:07,645 : INFO : serializing accumulator to return to master... 2018-03-26 13:57:07,650 : INFO : accumulator serialized 2018-03-26 13:57:08,397 : INFO : serializing accumulator to return to master... 2018-03-26 13:57:08,402 : INFO : accumulator serialized 2018-03-26 13:57:08,576 : INFO : 3 accumulators retrieved from output queue 2018-03-26 13:57:08,655 : INFO : accumulated word occurrence stats for 24866218 virtual documents 2018-03-26 13:57:09,172 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_20 2018-03-26 13:57:09,461 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_20.expElogbeta.npy with mmap=None 2018-03-26 13:57:09,545 : INFO : setting ignored attribute state to None 2018-03-26 13:57:09,546 : INFO : setting ignored attribute dispatcher to None 2018-03-26 13:57:09,546 : INFO : setting ignored attribute id2word to None 2018-03-26 13:57:09,547 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_20 2018-03-26 13:57:09,548 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_20.state 2018-03-26 13:57:09,722 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_20.state 2018-03-26 13:57:09,775 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 13:57:09,883 : INFO : 1 batches submitted to accumulate stats from 64 documents (190785 virtual) 2018-03-26 13:57:09,939 : INFO : 2 batches submitted to accumulate stats from 128 documents (350115 virtual) 2018-03-26 13:57:10,003 : INFO : 3 batches submitted to accumulate stats from 192 documents (497820 virtual) 2018-03-26 13:57:10,170 : INFO : 4 batches submitted to accumulate stats from 256 documents (793299 virtual) 2018-03-26 13:57:10,265 : INFO : 5 batches submitted to accumulate stats from 320 documents (960042 virtual) 2018-03-26 13:57:10,360 : INFO : 6 batches submitted to accumulate stats from 384 documents (1114698 virtual) 2018-03-26 13:57:16,416 : INFO : 7 batches submitted to accumulate stats from 448 documents (1262215 virtual) 2018-03-26 13:57:16,523 : INFO : 8 batches submitted to accumulate stats from 512 documents (1444627 virtual) 2018-03-26 13:57:18,033 : INFO : 9 batches submitted to accumulate stats from 576 documents (1623336 virtual) 2018-03-26 13:57:23,272 : INFO : 10 batches submitted to accumulate stats from 640 documents (1754795 virtual) 2018-03-26 13:57:24,317 : INFO : 11 batches submitted to accumulate stats from 704 documents (1930466 virtual) 2018-03-26 13:57:28,404 : INFO : 12 batches submitted to accumulate stats from 768 documents (2094686 virtual) 2018-03-26 13:57:29,243 : INFO : 13 batches submitted to accumulate stats from 832 documents (2254597 virtual) 2018-03-26 13:57:31,839 : INFO : 14 batches submitted to accumulate stats from 896 documents (2368318 virtual) 2018-03-26 13:57:34,994 : INFO : 15 batches submitted to accumulate stats from 960 documents (2503989 virtual) 2018-03-26 13:57:35,881 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2870436 virtual) 2018-03-26 13:57:39,351 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3046058 virtual) 2018-03-26 13:57:41,699 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3223862 virtual) 2018-03-26 13:57:42,399 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3447427 virtual) 2018-03-26 13:57:44,172 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3627563 virtual) 2018-03-26 13:57:48,286 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3790690 virtual) 2018-03-26 13:57:51,669 : INFO : 22 batches submitted to accumulate stats from 1408 documents (3969115 virtual) 2018-03-26 13:57:55,671 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4791388 virtual) 2018-03-26 13:57:58,228 : INFO : 24 batches submitted to accumulate stats from 1536 documents (4930529 virtual) 2018-03-26 13:58:01,084 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5147230 virtual) 2018-03-26 13:58:03,145 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5281132 virtual) 2018-03-26 13:58:04,816 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5486958 virtual) 2018-03-26 13:58:08,538 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5636210 virtual) 2018-03-26 13:58:10,348 : INFO : 29 batches submitted to accumulate stats from 1856 documents (5862642 virtual) 2018-03-26 13:58:16,028 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6060659 virtual) 2018-03-26 13:58:17,352 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6205164 virtual) 2018-03-26 13:58:23,631 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7801847 virtual) 2018-03-26 13:58:25,133 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8231804 virtual) 2018-03-26 13:58:32,493 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8404159 virtual) 2018-03-26 13:58:33,773 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8635278 virtual) 2018-03-26 13:58:36,497 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8781792 virtual) 2018-03-26 13:58:38,280 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8909101 virtual) 2018-03-26 13:58:45,369 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9094182 virtual) 2018-03-26 13:58:51,897 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9239207 virtual) 2018-03-26 13:58:54,608 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9401314 virtual) 2018-03-26 13:58:57,971 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9565604 virtual) 2018-03-26 13:58:59,562 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9721933 virtual) 2018-03-26 13:59:05,377 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9932544 virtual) 2018-03-26 13:59:05,640 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10102497 virtual) 2018-03-26 13:59:12,110 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10260164 virtual) 2018-03-26 13:59:12,568 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10402970 virtual) 2018-03-26 13:59:18,503 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10543308 virtual) 2018-03-26 13:59:20,908 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10698288 virtual) 2018-03-26 13:59:25,242 : INFO : 49 batches submitted to accumulate stats from 3136 documents (10852708 virtual) 2018-03-26 13:59:27,220 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11062859 virtual) 2018-03-26 13:59:31,228 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11182017 virtual) 2018-03-26 13:59:32,751 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11307644 virtual) 2018-03-26 13:59:35,613 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11504525 virtual) 2018-03-26 13:59:37,291 : INFO : 54 batches submitted to accumulate stats from 3456 documents (11655689 virtual) 2018-03-26 13:59:39,111 : INFO : 55 batches submitted to accumulate stats from 3520 documents (11809422 virtual) 2018-03-26 13:59:42,220 : INFO : 56 batches submitted to accumulate stats from 3584 documents (11978549 virtual) 2018-03-26 13:59:44,262 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12134254 virtual) 2018-03-26 13:59:44,428 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12308989 virtual) 2018-03-26 13:59:50,213 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12429976 virtual) 2018-03-26 13:59:51,149 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12613357 virtual) 2018-03-26 13:59:51,254 : INFO : 61 batches submitted to accumulate stats from 3904 documents (12808013 virtual) 2018-03-26 13:59:57,345 : INFO : 62 batches submitted to accumulate stats from 3968 documents (12935075 virtual) 2018-03-26 13:59:57,427 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13078161 virtual) 2018-03-26 13:59:58,542 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13207958 virtual) 2018-03-26 14:00:02,421 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13335490 virtual) 2018-03-26 14:00:04,901 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13478923 virtual) 2018-03-26 14:00:06,828 : INFO : 67 batches submitted to accumulate stats from 4288 documents (13644496 virtual) 2018-03-26 14:00:07,763 : INFO : 68 batches submitted to accumulate stats from 4352 documents (13802266 virtual) 2018-03-26 14:00:10,708 : INFO : 69 batches submitted to accumulate stats from 4416 documents (13978788 virtual) 2018-03-26 14:00:12,032 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14089142 virtual) 2018-03-26 14:00:13,188 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14336547 virtual) 2018-03-26 14:00:16,899 : INFO : 72 batches submitted to accumulate stats from 4608 documents (14548378 virtual) 2018-03-26 14:00:19,117 : INFO : 73 batches submitted to accumulate stats from 4672 documents (14699639 virtual) 2018-03-26 14:00:19,584 : INFO : 74 batches submitted to accumulate stats from 4736 documents (14866559 virtual) 2018-03-26 14:00:23,890 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15049173 virtual) 2018-03-26 14:00:23,982 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15196542 virtual) 2018-03-26 14:00:28,949 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15324172 virtual) 2018-03-26 14:00:29,871 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15468199 virtual) 2018-03-26 14:00:32,826 : INFO : 79 batches submitted to accumulate stats from 5056 documents (15676305 virtual) 2018-03-26 14:00:35,705 : INFO : 80 batches submitted to accumulate stats from 5120 documents (15787008 virtual) 2018-03-26 14:00:37,507 : INFO : 81 batches submitted to accumulate stats from 5184 documents (15939142 virtual) 2018-03-26 14:00:39,106 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16097226 virtual) 2018-03-26 14:00:41,268 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16301694 virtual) 2018-03-26 14:00:43,800 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16429156 virtual) 2018-03-26 14:00:45,717 : INFO : 85 batches submitted to accumulate stats from 5440 documents (16711197 virtual) 2018-03-26 14:00:47,542 : INFO : 86 batches submitted to accumulate stats from 5504 documents (16878235 virtual) 2018-03-26 14:00:50,377 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17123492 virtual) 2018-03-26 14:00:52,115 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17262709 virtual) 2018-03-26 14:00:55,486 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17385548 virtual) 2018-03-26 14:00:55,758 : INFO : 90 batches submitted to accumulate stats from 5760 documents (17536255 virtual) 2018-03-26 14:01:02,395 : INFO : 91 batches submitted to accumulate stats from 5824 documents (17722777 virtual) 2018-03-26 14:01:02,880 : INFO : 92 batches submitted to accumulate stats from 5888 documents (17875432 virtual) 2018-03-26 14:01:04,793 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18045093 virtual) 2018-03-26 14:01:07,983 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18310693 virtual) 2018-03-26 14:01:08,149 : INFO : 95 batches submitted to accumulate stats from 6080 documents (18435497 virtual) 2018-03-26 14:01:11,346 : INFO : 96 batches submitted to accumulate stats from 6144 documents (18582059 virtual) 2018-03-26 14:01:14,398 : INFO : 97 batches submitted to accumulate stats from 6208 documents (18757718 virtual) 2018-03-26 14:01:15,840 : INFO : 98 batches submitted to accumulate stats from 6272 documents (18869203 virtual) 2018-03-26 14:01:18,073 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19021695 virtual) 2018-03-26 14:01:21,098 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19217642 virtual) 2018-03-26 14:01:24,182 : INFO : 101 batches submitted to accumulate stats from 6464 documents (19426284 virtual) 2018-03-26 14:01:25,527 : INFO : 102 batches submitted to accumulate stats from 6528 documents (19645513 virtual) 2018-03-26 14:01:28,212 : INFO : 103 batches submitted to accumulate stats from 6592 documents (19809489 virtual) 2018-03-26 14:01:28,770 : INFO : 104 batches submitted to accumulate stats from 6656 documents (19985429 virtual) 2018-03-26 14:01:31,837 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20123144 virtual) 2018-03-26 14:01:36,612 : INFO : 106 batches submitted to accumulate stats from 6784 documents (21684977 virtual) 2018-03-26 14:01:37,072 : INFO : 107 batches submitted to accumulate stats from 6848 documents (21801196 virtual) 2018-03-26 14:01:40,821 : INFO : 108 batches submitted to accumulate stats from 6912 documents (21966275 virtual) 2018-03-26 14:01:43,780 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22129494 virtual) 2018-03-26 14:01:44,963 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22276624 virtual) 2018-03-26 14:01:47,350 : INFO : 111 batches submitted to accumulate stats from 7104 documents (22479374 virtual) 2018-03-26 14:01:50,160 : INFO : 112 batches submitted to accumulate stats from 7168 documents (22741725 virtual) 2018-03-26 14:01:54,078 : INFO : 113 batches submitted to accumulate stats from 7232 documents (22955004 virtual) 2018-03-26 14:01:56,986 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23196883 virtual) 2018-03-26 14:02:00,296 : INFO : 115 batches submitted to accumulate stats from 7360 documents (23351817 virtual) 2018-03-26 14:02:04,099 : INFO : 116 batches submitted to accumulate stats from 7424 documents (23532228 virtual) 2018-03-26 14:02:10,912 : INFO : 117 batches submitted to accumulate stats from 7488 documents (23695002 virtual) 2018-03-26 14:02:12,726 : INFO : 118 batches submitted to accumulate stats from 7552 documents (23888550 virtual) 2018-03-26 14:02:19,017 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24072199 virtual) 2018-03-26 14:02:20,320 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24220366 virtual) 2018-03-26 14:02:26,502 : INFO : 121 batches submitted to accumulate stats from 7744 documents (24303591 virtual) 2018-03-26 14:02:26,626 : INFO : 122 batches submitted to accumulate stats from 7808 documents (24467423 virtual) 2018-03-26 14:02:33,921 : INFO : 123 batches submitted to accumulate stats from 7872 documents (24634680 virtual) 2018-03-26 14:02:34,588 : INFO : 124 batches submitted to accumulate stats from 7936 documents (24759037 virtual) 2018-03-26 14:02:45,973 : INFO : serializing accumulator to return to master... 2018-03-26 14:02:45,979 : INFO : accumulator serialized 2018-03-26 14:02:46,478 : INFO : serializing accumulator to return to master... 2018-03-26 14:02:46,482 : INFO : accumulator serialized 2018-03-26 14:02:48,824 : INFO : serializing accumulator to return to master... 2018-03-26 14:02:48,829 : INFO : accumulator serialized 2018-03-26 14:02:49,065 : INFO : 3 accumulators retrieved from output queue 2018-03-26 14:02:49,195 : INFO : accumulated word occurrence stats for 24866259 virtual documents 2018-03-26 14:02:49,875 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_25 2018-03-26 14:02:50,159 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-26 14:02:50,267 : INFO : setting ignored attribute state to None 2018-03-26 14:02:50,268 : INFO : setting ignored attribute dispatcher to None 2018-03-26 14:02:50,269 : INFO : setting ignored attribute id2word to None 2018-03-26 14:02:50,269 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_25 2018-03-26 14:02:50,270 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_25.state 2018-03-26 14:02:50,484 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_25.state 2018-03-26 14:02:50,534 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 14:02:50,645 : INFO : 1 batches submitted to accumulate stats from 64 documents (190779 virtual) 2018-03-26 14:02:50,703 : INFO : 2 batches submitted to accumulate stats from 128 documents (350160 virtual) 2018-03-26 14:02:50,775 : INFO : 3 batches submitted to accumulate stats from 192 documents (501387 virtual) 2018-03-26 14:02:50,943 : INFO : 4 batches submitted to accumulate stats from 256 documents (796879 virtual) 2018-03-26 14:02:51,045 : INFO : 5 batches submitted to accumulate stats from 320 documents (961646 virtual) 2018-03-26 14:02:51,139 : INFO : 6 batches submitted to accumulate stats from 384 documents (1114815 virtual) 2018-03-26 14:02:57,856 : INFO : 7 batches submitted to accumulate stats from 448 documents (1262309 virtual) 2018-03-26 14:02:57,971 : INFO : 8 batches submitted to accumulate stats from 512 documents (1456497 virtual) 2018-03-26 14:02:59,742 : INFO : 9 batches submitted to accumulate stats from 576 documents (1624989 virtual) 2018-03-26 14:03:05,460 : INFO : 10 batches submitted to accumulate stats from 640 documents (1758128 virtual) 2018-03-26 14:03:06,749 : INFO : 11 batches submitted to accumulate stats from 704 documents (1936965 virtual) 2018-03-26 14:03:11,816 : INFO : 12 batches submitted to accumulate stats from 768 documents (2094796 virtual) 2018-03-26 14:03:11,959 : INFO : 13 batches submitted to accumulate stats from 832 documents (2262017 virtual) 2018-03-26 14:03:15,706 : INFO : 14 batches submitted to accumulate stats from 896 documents (2368610 virtual) 2018-03-26 14:03:18,331 : INFO : 15 batches submitted to accumulate stats from 960 documents (2524061 virtual) 2018-03-26 14:03:19,543 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2871969 virtual) 2018-03-26 14:03:24,290 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3057819 virtual) 2018-03-26 14:03:25,505 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3236661 virtual) 2018-03-26 14:03:26,996 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3452650 virtual) 2018-03-26 14:03:28,944 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3633825 virtual) 2018-03-26 14:03:32,990 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3797767 virtual) 2018-03-26 14:03:37,388 : INFO : 22 batches submitted to accumulate stats from 1408 documents (3980018 virtual) 2018-03-26 14:03:40,937 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4792531 virtual) 2018-03-26 14:03:43,017 : INFO : 24 batches submitted to accumulate stats from 1536 documents (4932182 virtual) 2018-03-26 14:03:47,994 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5158482 virtual) 2018-03-26 14:03:49,594 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5281089 virtual) 2018-03-26 14:03:50,640 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5490266 virtual) 2018-03-26 14:03:56,364 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5636177 virtual) 2018-03-26 14:03:56,638 : INFO : 29 batches submitted to accumulate stats from 1856 documents (5862626 virtual) 2018-03-26 14:04:02,042 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6060716 virtual) 2018-03-26 14:04:05,941 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6208143 virtual) 2018-03-26 14:04:12,183 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7807106 virtual) 2018-03-26 14:04:12,731 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8240097 virtual) 2018-03-26 14:04:21,869 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8476102 virtual) 2018-03-26 14:04:22,379 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8635367 virtual) 2018-03-26 14:04:26,992 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8781892 virtual) 2018-03-26 14:04:28,320 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8909138 virtual) 2018-03-26 14:04:39,038 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9097053 virtual) 2018-03-26 14:04:43,019 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9239254 virtual) 2018-03-26 14:04:46,348 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9405822 virtual) 2018-03-26 14:04:49,792 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9565637 virtual) 2018-03-26 14:04:51,918 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9721905 virtual) 2018-03-26 14:04:58,249 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9932516 virtual) 2018-03-26 14:04:58,490 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10102469 virtual) 2018-03-26 14:05:05,847 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10260972 virtual) 2018-03-26 14:05:06,158 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10403045 virtual) 2018-03-26 14:05:13,009 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10543490 virtual) 2018-03-26 14:05:15,352 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10698326 virtual) 2018-03-26 14:05:20,524 : INFO : 49 batches submitted to accumulate stats from 3136 documents (10867384 virtual) 2018-03-26 14:05:22,230 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11074123 virtual) 2018-03-26 14:05:27,016 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11190356 virtual) 2018-03-26 14:05:28,430 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11307685 virtual) 2018-03-26 14:05:33,644 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11504499 virtual) 2018-03-26 14:05:33,866 : INFO : 54 batches submitted to accumulate stats from 3456 documents (11655664 virtual) 2018-03-26 14:05:36,093 : INFO : 55 batches submitted to accumulate stats from 3520 documents (11809397 virtual) 2018-03-26 14:05:39,331 : INFO : 56 batches submitted to accumulate stats from 3584 documents (11978524 virtual) 2018-03-26 14:05:41,481 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12134229 virtual) 2018-03-26 14:05:42,866 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12307812 virtual) 2018-03-26 14:05:47,675 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12429943 virtual) 2018-03-26 14:05:48,908 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12613324 virtual) 2018-03-26 14:05:50,577 : INFO : 61 batches submitted to accumulate stats from 3904 documents (12807888 virtual) 2018-03-26 14:05:55,607 : INFO : 62 batches submitted to accumulate stats from 3968 documents (12930188 virtual) 2018-03-26 14:05:56,413 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13070005 virtual) 2018-03-26 14:05:58,479 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13207915 virtual) 2018-03-26 14:06:01,364 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13335447 virtual) 2018-03-26 14:06:04,818 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13478776 virtual) 2018-03-26 14:06:06,922 : INFO : 67 batches submitted to accumulate stats from 4288 documents (13635398 virtual) 2018-03-26 14:06:07,596 : INFO : 68 batches submitted to accumulate stats from 4352 documents (13802153 virtual) 2018-03-26 14:06:10,958 : INFO : 69 batches submitted to accumulate stats from 4416 documents (13977086 virtual) 2018-03-26 14:06:13,187 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14086833 virtual) 2018-03-26 14:06:13,847 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14332591 virtual) 2018-03-26 14:06:17,205 : INFO : 72 batches submitted to accumulate stats from 4608 documents (14548171 virtual) 2018-03-26 14:06:20,695 : INFO : 73 batches submitted to accumulate stats from 4672 documents (14699358 virtual) 2018-03-26 14:06:21,633 : INFO : 74 batches submitted to accumulate stats from 4736 documents (14866175 virtual) 2018-03-26 14:06:25,098 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15040924 virtual) 2018-03-26 14:06:25,907 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15192980 virtual) 2018-03-26 14:06:31,876 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15310713 virtual) 2018-03-26 14:06:32,478 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15452068 virtual) 2018-03-26 14:06:35,073 : INFO : 79 batches submitted to accumulate stats from 5056 documents (15667042 virtual) 2018-03-26 14:06:39,323 : INFO : 80 batches submitted to accumulate stats from 5120 documents (15775120 virtual) 2018-03-26 14:06:40,478 : INFO : 81 batches submitted to accumulate stats from 5184 documents (15919040 virtual) 2018-03-26 14:06:42,344 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16091902 virtual) 2018-03-26 14:06:44,911 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16287575 virtual) 2018-03-26 14:06:47,267 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16426071 virtual) 2018-03-26 14:06:49,792 : INFO : 85 batches submitted to accumulate stats from 5440 documents (16698931 virtual) 2018-03-26 14:06:51,856 : INFO : 86 batches submitted to accumulate stats from 5504 documents (16867068 virtual) 2018-03-26 14:06:54,222 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17111840 virtual) 2018-03-26 14:06:57,367 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17257544 virtual) 2018-03-26 14:07:00,340 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17375920 virtual) 2018-03-26 14:07:00,822 : INFO : 90 batches submitted to accumulate stats from 5760 documents (17531053 virtual) 2018-03-26 14:07:08,111 : INFO : 91 batches submitted to accumulate stats from 5824 documents (17714635 virtual) 2018-03-26 14:07:08,509 : INFO : 92 batches submitted to accumulate stats from 5888 documents (17856778 virtual) 2018-03-26 14:07:10,750 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18026374 virtual) 2018-03-26 14:07:13,887 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18305239 virtual) 2018-03-26 14:07:14,844 : INFO : 95 batches submitted to accumulate stats from 6080 documents (18435087 virtual) 2018-03-26 14:07:17,929 : INFO : 96 batches submitted to accumulate stats from 6144 documents (18568201 virtual) 2018-03-26 14:07:21,638 : INFO : 97 batches submitted to accumulate stats from 6208 documents (18746418 virtual) 2018-03-26 14:07:22,166 : INFO : 98 batches submitted to accumulate stats from 6272 documents (18858999 virtual) 2018-03-26 14:07:25,556 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19015929 virtual) 2018-03-26 14:07:28,108 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19216105 virtual) 2018-03-26 14:07:31,652 : INFO : 101 batches submitted to accumulate stats from 6464 documents (19414990 virtual) 2018-03-26 14:07:35,195 : INFO : 102 batches submitted to accumulate stats from 6528 documents (19635994 virtual) 2018-03-26 14:07:35,950 : INFO : 103 batches submitted to accumulate stats from 6592 documents (19803055 virtual) 2018-03-26 14:07:36,944 : INFO : 104 batches submitted to accumulate stats from 6656 documents (19934423 virtual) 2018-03-26 14:07:42,246 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20106699 virtual) 2018-03-26 14:07:45,594 : INFO : 106 batches submitted to accumulate stats from 6784 documents (21672407 virtual) 2018-03-26 14:07:45,720 : INFO : 107 batches submitted to accumulate stats from 6848 documents (21790075 virtual) 2018-03-26 14:07:52,084 : INFO : 108 batches submitted to accumulate stats from 6912 documents (21951491 virtual) 2018-03-26 14:07:52,724 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22124196 virtual) 2018-03-26 14:07:53,427 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22260175 virtual) 2018-03-26 14:07:58,604 : INFO : 111 batches submitted to accumulate stats from 7104 documents (22460669 virtual) 2018-03-26 14:08:00,003 : INFO : 112 batches submitted to accumulate stats from 7168 documents (22700527 virtual) 2018-03-26 14:08:05,622 : INFO : 113 batches submitted to accumulate stats from 7232 documents (22952579 virtual) 2018-03-26 14:08:07,943 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23191707 virtual) 2018-03-26 14:08:12,103 : INFO : 115 batches submitted to accumulate stats from 7360 documents (23340034 virtual) 2018-03-26 14:08:15,733 : INFO : 116 batches submitted to accumulate stats from 7424 documents (23523581 virtual) 2018-03-26 14:08:23,199 : INFO : 117 batches submitted to accumulate stats from 7488 documents (23678219 virtual) 2018-03-26 14:08:26,595 : INFO : 118 batches submitted to accumulate stats from 7552 documents (23857336 virtual) 2018-03-26 14:08:33,149 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24063139 virtual) 2018-03-26 14:08:33,241 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24210134 virtual) 2018-03-26 14:08:39,546 : INFO : 121 batches submitted to accumulate stats from 7744 documents (24295181 virtual) 2018-03-26 14:08:41,696 : INFO : 122 batches submitted to accumulate stats from 7808 documents (24462826 virtual) 2018-03-26 14:08:47,674 : INFO : 123 batches submitted to accumulate stats from 7872 documents (24621211 virtual) 2018-03-26 14:08:50,750 : INFO : 124 batches submitted to accumulate stats from 7936 documents (24758502 virtual) 2018-03-26 14:09:02,227 : INFO : serializing accumulator to return to master... 2018-03-26 14:09:02,234 : INFO : accumulator serialized 2018-03-26 14:09:03,419 : INFO : serializing accumulator to return to master... 2018-03-26 14:09:03,424 : INFO : accumulator serialized 2018-03-26 14:09:06,167 : INFO : serializing accumulator to return to master... 2018-03-26 14:09:06,172 : INFO : accumulator serialized 2018-03-26 14:09:06,446 : INFO : 3 accumulators retrieved from output queue 2018-03-26 14:09:06,629 : INFO : accumulated word occurrence stats for 24866263 virtual documents 2018-03-26 14:09:07,478 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_30 2018-03-26 14:09:07,527 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-26 14:09:07,637 : INFO : setting ignored attribute state to None 2018-03-26 14:09:07,638 : INFO : setting ignored attribute dispatcher to None 2018-03-26 14:09:07,638 : INFO : setting ignored attribute id2word to None 2018-03-26 14:09:07,639 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_30 2018-03-26 14:09:07,640 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_30.state 2018-03-26 14:09:07,893 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_30.state 2018-03-26 14:09:07,954 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 14:09:08,065 : INFO : 1 batches submitted to accumulate stats from 64 documents (190785 virtual) 2018-03-26 14:09:08,123 : INFO : 2 batches submitted to accumulate stats from 128 documents (350115 virtual) 2018-03-26 14:09:08,187 : INFO : 3 batches submitted to accumulate stats from 192 documents (497820 virtual) 2018-03-26 14:09:08,357 : INFO : 4 batches submitted to accumulate stats from 256 documents (793299 virtual) 2018-03-26 14:09:08,461 : INFO : 5 batches submitted to accumulate stats from 320 documents (960042 virtual) 2018-03-26 14:09:08,553 : INFO : 6 batches submitted to accumulate stats from 384 documents (1109608 virtual) 2018-03-26 14:09:15,385 : INFO : 7 batches submitted to accumulate stats from 448 documents (1258779 virtual) 2018-03-26 14:09:15,740 : INFO : 8 batches submitted to accumulate stats from 512 documents (1444598 virtual) 2018-03-26 14:09:17,598 : INFO : 9 batches submitted to accumulate stats from 576 documents (1623239 virtual) 2018-03-26 14:09:23,985 : INFO : 10 batches submitted to accumulate stats from 640 documents (1754739 virtual) 2018-03-26 14:09:25,167 : INFO : 11 batches submitted to accumulate stats from 704 documents (1917819 virtual) 2018-03-26 14:09:29,795 : INFO : 12 batches submitted to accumulate stats from 768 documents (2083102 virtual) 2018-03-26 14:09:30,960 : INFO : 13 batches submitted to accumulate stats from 832 documents (2248568 virtual) 2018-03-26 14:09:34,401 : INFO : 14 batches submitted to accumulate stats from 896 documents (2363028 virtual) 2018-03-26 14:09:37,829 : INFO : 15 batches submitted to accumulate stats from 960 documents (2498649 virtual) 2018-03-26 14:09:38,469 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2858973 virtual) 2018-03-26 14:09:42,585 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3045929 virtual) 2018-03-26 14:09:45,899 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3223622 virtual) 2018-03-26 14:09:46,209 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3437212 virtual) 2018-03-26 14:09:48,062 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3622984 virtual) 2018-03-26 14:09:52,680 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3790451 virtual) 2018-03-26 14:09:57,382 : INFO : 22 batches submitted to accumulate stats from 1408 documents (3967485 virtual) 2018-03-26 14:10:02,317 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4786151 virtual) 2018-03-26 14:10:04,319 : INFO : 24 batches submitted to accumulate stats from 1536 documents (4930065 virtual) 2018-03-26 14:10:07,813 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5145424 virtual) 2018-03-26 14:10:11,603 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5281006 virtual) 2018-03-26 14:10:12,005 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5482865 virtual) 2018-03-26 14:10:16,687 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5628541 virtual) 2018-03-26 14:10:18,752 : INFO : 29 batches submitted to accumulate stats from 1856 documents (5851878 virtual) 2018-03-26 14:10:25,294 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6054053 virtual) 2018-03-26 14:10:26,714 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6190635 virtual) 2018-03-26 14:10:34,092 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7797467 virtual) 2018-03-26 14:10:35,534 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8231446 virtual) 2018-03-26 14:10:44,123 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8386171 virtual) 2018-03-26 14:10:45,731 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8612730 virtual) 2018-03-26 14:10:49,892 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8773535 virtual) 2018-03-26 14:10:50,875 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8902028 virtual) 2018-03-26 14:10:58,454 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9078729 virtual) 2018-03-26 14:11:08,090 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9225985 virtual) 2018-03-26 14:11:08,990 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9387615 virtual) 2018-03-26 14:11:15,008 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9553697 virtual) 2018-03-26 14:11:15,968 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9701632 virtual) 2018-03-26 14:11:23,220 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9915588 virtual) 2018-03-26 14:11:23,416 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10088495 virtual) 2018-03-26 14:11:31,375 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10231712 virtual) 2018-03-26 14:11:31,597 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10377068 virtual) 2018-03-26 14:11:38,542 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10517122 virtual) 2018-03-26 14:11:41,709 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10683724 virtual) 2018-03-26 14:11:46,887 : INFO : 49 batches submitted to accumulate stats from 3136 documents (10828283 virtual) 2018-03-26 14:11:47,942 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11035126 virtual) 2018-03-26 14:11:53,613 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11178730 virtual) 2018-03-26 14:11:54,845 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11301894 virtual) 2018-03-26 14:11:58,105 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11481062 virtual) 2018-03-26 14:12:01,383 : INFO : 54 batches submitted to accumulate stats from 3456 documents (11630429 virtual) 2018-03-26 14:12:01,736 : INFO : 55 batches submitted to accumulate stats from 3520 documents (11760701 virtual) 2018-03-26 14:12:07,744 : INFO : 56 batches submitted to accumulate stats from 3584 documents (11939701 virtual) 2018-03-26 14:12:08,183 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12096505 virtual) 2018-03-26 14:12:08,554 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12257358 virtual) 2018-03-26 14:12:14,991 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12396847 virtual) 2018-03-26 14:12:15,632 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12568770 virtual) 2018-03-26 14:12:15,820 : INFO : 61 batches submitted to accumulate stats from 3904 documents (12783032 virtual) 2018-03-26 14:12:23,133 : INFO : 62 batches submitted to accumulate stats from 3968 documents (12912740 virtual) 2018-03-26 14:12:23,678 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13038484 virtual) 2018-03-26 14:12:23,912 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13170268 virtual) 2018-03-26 14:12:29,947 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13309381 virtual) 2018-03-26 14:12:32,052 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13450809 virtual) 2018-03-26 14:12:34,187 : INFO : 67 batches submitted to accumulate stats from 4288 documents (13593876 virtual) 2018-03-26 14:12:36,467 : INFO : 68 batches submitted to accumulate stats from 4352 documents (13784612 virtual) 2018-03-26 14:12:38,341 : INFO : 69 batches submitted to accumulate stats from 4416 documents (13932587 virtual) 2018-03-26 14:12:40,602 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14062690 virtual) 2018-03-26 14:12:43,920 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14280026 virtual) 2018-03-26 14:12:45,900 : INFO : 72 batches submitted to accumulate stats from 4608 documents (14418475 virtual) 2018-03-26 14:12:47,926 : INFO : 73 batches submitted to accumulate stats from 4672 documents (14644323 virtual) 2018-03-26 14:12:52,888 : INFO : 74 batches submitted to accumulate stats from 4736 documents (14821707 virtual) 2018-03-26 14:12:53,263 : INFO : 75 batches submitted to accumulate stats from 4800 documents (14993227 virtual) 2018-03-26 14:12:54,818 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15164993 virtual) 2018-03-26 14:12:59,661 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15279453 virtual) 2018-03-26 14:13:02,595 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15425545 virtual) 2018-03-26 14:13:05,348 : INFO : 79 batches submitted to accumulate stats from 5056 documents (15607331 virtual) 2018-03-26 14:13:08,023 : INFO : 80 batches submitted to accumulate stats from 5120 documents (15754553 virtual) 2018-03-26 14:13:10,976 : INFO : 81 batches submitted to accumulate stats from 5184 documents (15886817 virtual) 2018-03-26 14:13:13,783 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16036515 virtual) 2018-03-26 14:13:13,898 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16224075 virtual) 2018-03-26 14:13:18,347 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16382347 virtual) 2018-03-26 14:13:21,041 : INFO : 85 batches submitted to accumulate stats from 5440 documents (16542906 virtual) 2018-03-26 14:13:22,496 : INFO : 86 batches submitted to accumulate stats from 5504 documents (16810105 virtual) 2018-03-26 14:13:25,133 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17042296 virtual) 2018-03-26 14:13:28,353 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17225673 virtual) 2018-03-26 14:13:31,429 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17348015 virtual) 2018-03-26 14:13:32,769 : INFO : 90 batches submitted to accumulate stats from 5760 documents (17487268 virtual) 2018-03-26 14:13:36,233 : INFO : 91 batches submitted to accumulate stats from 5824 documents (17626846 virtual) 2018-03-26 14:13:43,068 : INFO : 92 batches submitted to accumulate stats from 5888 documents (17820655 virtual) 2018-03-26 14:13:43,318 : INFO : 93 batches submitted to accumulate stats from 5952 documents (17974230 virtual) 2018-03-26 14:13:44,981 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18281702 virtual) 2018-03-26 14:13:48,963 : INFO : 95 batches submitted to accumulate stats from 6080 documents (18395681 virtual) 2018-03-26 14:13:50,241 : INFO : 96 batches submitted to accumulate stats from 6144 documents (18517133 virtual) 2018-03-26 14:13:51,765 : INFO : 97 batches submitted to accumulate stats from 6208 documents (18700772 virtual) 2018-03-26 14:13:57,531 : INFO : 98 batches submitted to accumulate stats from 6272 documents (18816773 virtual) 2018-03-26 14:13:58,772 : INFO : 99 batches submitted to accumulate stats from 6336 documents (18925609 virtual) 2018-03-26 14:14:03,256 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19157936 virtual) 2018-03-26 14:14:04,822 : INFO : 101 batches submitted to accumulate stats from 6464 documents (19348887 virtual) 2018-03-26 14:14:07,088 : INFO : 102 batches submitted to accumulate stats from 6528 documents (19537159 virtual) 2018-03-26 14:14:10,434 : INFO : 103 batches submitted to accumulate stats from 6592 documents (19742564 virtual) 2018-03-26 14:14:12,305 : INFO : 104 batches submitted to accumulate stats from 6656 documents (19889963 virtual) 2018-03-26 14:14:12,738 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20044935 virtual) 2018-03-26 14:14:21,262 : INFO : 106 batches submitted to accumulate stats from 6784 documents (20174341 virtual) 2018-03-26 14:14:22,114 : INFO : 107 batches submitted to accumulate stats from 6848 documents (21752421 virtual) 2018-03-26 14:14:22,206 : INFO : 108 batches submitted to accumulate stats from 6912 documents (21896259 virtual) 2018-03-26 14:14:28,788 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22067689 virtual) 2018-03-26 14:14:29,726 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22207092 virtual) 2018-03-26 14:14:30,979 : INFO : 111 batches submitted to accumulate stats from 7104 documents (22351685 virtual) 2018-03-26 14:14:35,013 : INFO : 112 batches submitted to accumulate stats from 7168 documents (22551060 virtual) 2018-03-26 14:14:37,429 : INFO : 113 batches submitted to accumulate stats from 7232 documents (22858489 virtual) 2018-03-26 14:14:43,535 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23002554 virtual) 2018-03-26 14:14:44,391 : INFO : 115 batches submitted to accumulate stats from 7360 documents (23280952 virtual) 2018-03-26 14:14:50,614 : INFO : 116 batches submitted to accumulate stats from 7424 documents (23447058 virtual) 2018-03-26 14:14:52,616 : INFO : 117 batches submitted to accumulate stats from 7488 documents (23613528 virtual) 2018-03-26 14:14:58,838 : INFO : 118 batches submitted to accumulate stats from 7552 documents (23819919 virtual) 2018-03-26 14:15:05,374 : INFO : 119 batches submitted to accumulate stats from 7616 documents (23949486 virtual) 2018-03-26 14:15:11,175 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24168253 virtual) 2018-03-26 14:15:13,232 : INFO : 121 batches submitted to accumulate stats from 7744 documents (24260220 virtual) 2018-03-26 14:15:19,389 : INFO : 122 batches submitted to accumulate stats from 7808 documents (24389356 virtual) 2018-03-26 14:15:23,777 : INFO : 123 batches submitted to accumulate stats from 7872 documents (24543450 virtual) 2018-03-26 14:15:26,171 : INFO : 124 batches submitted to accumulate stats from 7936 documents (24718987 virtual) 2018-03-26 14:15:30,475 : INFO : 125 batches submitted to accumulate stats from 8000 documents (24755656 virtual) 2018-03-26 14:15:42,754 : INFO : serializing accumulator to return to master... 2018-03-26 14:15:42,761 : INFO : accumulator serialized 2018-03-26 14:15:43,754 : INFO : serializing accumulator to return to master... 2018-03-26 14:15:43,759 : INFO : accumulator serialized 2018-03-26 14:15:44,500 : INFO : serializing accumulator to return to master... 2018-03-26 14:15:44,505 : INFO : accumulator serialized 2018-03-26 14:15:44,891 : INFO : 3 accumulators retrieved from output queue 2018-03-26 14:15:45,185 : INFO : accumulated word occurrence stats for 24866293 virtual documents 2018-03-26 14:15:46,217 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_35 2018-03-26 14:15:46,268 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_35.expElogbeta.npy with mmap=None 2018-03-26 14:15:46,399 : INFO : setting ignored attribute state to None 2018-03-26 14:15:46,399 : INFO : setting ignored attribute dispatcher to None 2018-03-26 14:15:46,400 : INFO : setting ignored attribute id2word to None 2018-03-26 14:15:46,400 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_35 2018-03-26 14:15:46,401 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_35.state 2018-03-26 14:15:46,698 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_35.state 2018-03-26 14:15:46,760 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 14:15:46,872 : INFO : 1 batches submitted to accumulate stats from 64 documents (190785 virtual) 2018-03-26 14:15:46,931 : INFO : 2 batches submitted to accumulate stats from 128 documents (350115 virtual) 2018-03-26 14:15:46,999 : INFO : 3 batches submitted to accumulate stats from 192 documents (497820 virtual) 2018-03-26 14:15:47,182 : INFO : 4 batches submitted to accumulate stats from 256 documents (793299 virtual) 2018-03-26 14:15:47,288 : INFO : 5 batches submitted to accumulate stats from 320 documents (960042 virtual) 2018-03-26 14:15:47,384 : INFO : 6 batches submitted to accumulate stats from 384 documents (1109608 virtual) 2018-03-26 14:15:55,329 : INFO : 7 batches submitted to accumulate stats from 448 documents (1258779 virtual) 2018-03-26 14:15:55,447 : INFO : 8 batches submitted to accumulate stats from 512 documents (1444598 virtual) 2018-03-26 14:15:57,320 : INFO : 9 batches submitted to accumulate stats from 576 documents (1623239 virtual) 2018-03-26 14:16:04,237 : INFO : 10 batches submitted to accumulate stats from 640 documents (1754739 virtual) 2018-03-26 14:16:05,191 : INFO : 11 batches submitted to accumulate stats from 704 documents (1917819 virtual) 2018-03-26 14:16:11,348 : INFO : 12 batches submitted to accumulate stats from 768 documents (2083102 virtual) 2018-03-26 14:16:11,948 : INFO : 13 batches submitted to accumulate stats from 832 documents (2248568 virtual) 2018-03-26 14:16:15,010 : INFO : 14 batches submitted to accumulate stats from 896 documents (2363028 virtual) 2018-03-26 14:16:19,016 : INFO : 15 batches submitted to accumulate stats from 960 documents (2498649 virtual) 2018-03-26 14:16:21,091 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2858973 virtual) 2018-03-26 14:16:23,939 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3045929 virtual) 2018-03-26 14:16:27,704 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3223506 virtual) 2018-03-26 14:16:29,631 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3432839 virtual) 2018-03-26 14:16:30,062 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3618211 virtual) 2018-03-26 14:16:35,475 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3777175 virtual) 2018-03-26 14:16:39,875 : INFO : 22 batches submitted to accumulate stats from 1408 documents (3962918 virtual) 2018-03-26 14:16:45,085 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4782372 virtual) 2018-03-26 14:16:48,904 : INFO : 24 batches submitted to accumulate stats from 1536 documents (4921064 virtual) 2018-03-26 14:16:50,975 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5130593 virtual) 2018-03-26 14:16:55,378 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5277946 virtual) 2018-03-26 14:16:56,914 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5475234 virtual) 2018-03-26 14:17:00,969 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5615177 virtual) 2018-03-26 14:17:03,764 : INFO : 29 batches submitted to accumulate stats from 1856 documents (5849786 virtual) 2018-03-26 14:17:11,325 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6046850 virtual) 2018-03-26 14:17:11,414 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6185297 virtual) 2018-03-26 14:17:19,135 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7790045 virtual) 2018-03-26 14:17:22,411 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8229232 virtual) 2018-03-26 14:17:31,404 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8383755 virtual) 2018-03-26 14:17:33,019 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8610248 virtual) 2018-03-26 14:17:38,475 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8756998 virtual) 2018-03-26 14:17:39,059 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8891112 virtual) 2018-03-26 14:17:47,342 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9058029 virtual) 2018-03-26 14:17:58,342 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9209815 virtual) 2018-03-26 14:17:59,183 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9374777 virtual) 2018-03-26 14:18:06,097 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9547504 virtual) 2018-03-26 14:18:06,678 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9695383 virtual) 2018-03-26 14:18:14,685 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9893678 virtual) 2018-03-26 14:18:14,788 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10062968 virtual) 2018-03-26 14:18:23,913 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10220976 virtual) 2018-03-26 14:18:24,041 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10360320 virtual) 2018-03-26 14:18:31,846 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10486515 virtual) 2018-03-26 14:18:34,520 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10675712 virtual) 2018-03-26 14:18:40,576 : INFO : 49 batches submitted to accumulate stats from 3136 documents (10805840 virtual) 2018-03-26 14:18:42,392 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11026822 virtual) 2018-03-26 14:18:47,713 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11166557 virtual) 2018-03-26 14:18:49,361 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11292626 virtual) 2018-03-26 14:18:55,597 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11455943 virtual) 2018-03-26 14:18:56,188 : INFO : 54 batches submitted to accumulate stats from 3456 documents (11618709 virtual) 2018-03-26 14:18:57,403 : INFO : 55 batches submitted to accumulate stats from 3520 documents (11741789 virtual) 2018-03-26 14:19:03,593 : INFO : 56 batches submitted to accumulate stats from 3584 documents (11920191 virtual) 2018-03-26 14:19:04,078 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12077067 virtual) 2018-03-26 14:19:06,602 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12250763 virtual) 2018-03-26 14:19:11,884 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12381731 virtual) 2018-03-26 14:19:13,127 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12538397 virtual) 2018-03-26 14:19:13,260 : INFO : 61 batches submitted to accumulate stats from 3904 documents (12765154 virtual) 2018-03-26 14:19:21,675 : INFO : 62 batches submitted to accumulate stats from 3968 documents (12894462 virtual) 2018-03-26 14:19:21,788 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13014051 virtual) 2018-03-26 14:19:22,300 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13159103 virtual) 2018-03-26 14:19:28,542 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13287987 virtual) 2018-03-26 14:19:30,317 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13411772 virtual) 2018-03-26 14:19:34,308 : INFO : 67 batches submitted to accumulate stats from 4288 documents (13550131 virtual) 2018-03-26 14:19:35,693 : INFO : 68 batches submitted to accumulate stats from 4352 documents (13744466 virtual) 2018-03-26 14:19:36,810 : INFO : 69 batches submitted to accumulate stats from 4416 documents (13904005 virtual) 2018-03-26 14:19:41,930 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14051576 virtual) 2018-03-26 14:19:42,783 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14254429 virtual) 2018-03-26 14:19:43,779 : INFO : 72 batches submitted to accumulate stats from 4608 documents (14395237 virtual) 2018-03-26 14:19:49,718 : INFO : 73 batches submitted to accumulate stats from 4672 documents (14612812 virtual) 2018-03-26 14:19:52,000 : INFO : 74 batches submitted to accumulate stats from 4736 documents (14791103 virtual) 2018-03-26 14:19:53,268 : INFO : 75 batches submitted to accumulate stats from 4800 documents (14944104 virtual) 2018-03-26 14:19:57,876 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15100547 virtual) 2018-03-26 14:20:00,429 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15249555 virtual) 2018-03-26 14:20:02,012 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15396704 virtual) 2018-03-26 14:20:09,216 : INFO : 79 batches submitted to accumulate stats from 5056 documents (15561761 virtual) 2018-03-26 14:20:09,751 : INFO : 80 batches submitted to accumulate stats from 5120 documents (15734931 virtual) 2018-03-26 14:20:10,535 : INFO : 81 batches submitted to accumulate stats from 5184 documents (15865351 virtual) 2018-03-26 14:20:17,880 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16005328 virtual) 2018-03-26 14:20:18,001 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16183776 virtual) 2018-03-26 14:20:18,750 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16361902 virtual) 2018-03-26 14:20:26,020 : INFO : 85 batches submitted to accumulate stats from 5440 documents (16510656 virtual) 2018-03-26 14:20:26,721 : INFO : 86 batches submitted to accumulate stats from 5504 documents (16759411 virtual) 2018-03-26 14:20:27,418 : INFO : 87 batches submitted to accumulate stats from 5568 documents (16919854 virtual) 2018-03-26 14:20:33,593 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17173396 virtual) 2018-03-26 14:20:36,244 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17305260 virtual) 2018-03-26 14:20:36,818 : INFO : 90 batches submitted to accumulate stats from 5760 documents (17446471 virtual) 2018-03-26 14:20:42,314 : INFO : 91 batches submitted to accumulate stats from 5824 documents (17597610 virtual) 2018-03-26 14:20:45,779 : INFO : 92 batches submitted to accumulate stats from 5888 documents (17769146 virtual) 2018-03-26 14:20:49,635 : INFO : 93 batches submitted to accumulate stats from 5952 documents (17937914 virtual) 2018-03-26 14:20:52,902 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18210549 virtual) 2018-03-26 14:20:54,822 : INFO : 95 batches submitted to accumulate stats from 6080 documents (18357818 virtual) 2018-03-26 14:20:57,074 : INFO : 96 batches submitted to accumulate stats from 6144 documents (18482737 virtual) 2018-03-26 14:21:01,216 : INFO : 97 batches submitted to accumulate stats from 6208 documents (18639884 virtual) 2018-03-26 14:21:03,649 : INFO : 98 batches submitted to accumulate stats from 6272 documents (18782322 virtual) 2018-03-26 14:21:06,429 : INFO : 99 batches submitted to accumulate stats from 6336 documents (18904974 virtual) 2018-03-26 14:21:11,784 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19093327 virtual) 2018-03-26 14:21:13,082 : INFO : 101 batches submitted to accumulate stats from 6464 documents (19282041 virtual) 2018-03-26 14:21:15,734 : INFO : 102 batches submitted to accumulate stats from 6528 documents (19499852 virtual) 2018-03-26 14:21:20,554 : INFO : 103 batches submitted to accumulate stats from 6592 documents (19695816 virtual) 2018-03-26 14:21:20,687 : INFO : 104 batches submitted to accumulate stats from 6656 documents (19856206 virtual) 2018-03-26 14:21:22,765 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20008233 virtual) 2018-03-26 14:21:30,280 : INFO : 106 batches submitted to accumulate stats from 6784 documents (20140195 virtual) 2018-03-26 14:21:31,209 : INFO : 107 batches submitted to accumulate stats from 6848 documents (21730351 virtual) 2018-03-26 14:21:33,916 : INFO : 108 batches submitted to accumulate stats from 6912 documents (21833852 virtual) 2018-03-26 14:21:39,634 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22011960 virtual) 2018-03-26 14:21:40,472 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22175966 virtual) 2018-03-26 14:21:42,312 : INFO : 111 batches submitted to accumulate stats from 7104 documents (22309050 virtual) 2018-03-26 14:21:46,822 : INFO : 112 batches submitted to accumulate stats from 7168 documents (22502745 virtual) 2018-03-26 14:21:47,461 : INFO : 113 batches submitted to accumulate stats from 7232 documents (22797332 virtual) 2018-03-26 14:21:56,084 : INFO : 114 batches submitted to accumulate stats from 7296 documents (22971260 virtual) 2018-03-26 14:21:56,311 : INFO : 115 batches submitted to accumulate stats from 7360 documents (23236854 virtual) 2018-03-26 14:22:03,374 : INFO : 116 batches submitted to accumulate stats from 7424 documents (23383055 virtual) 2018-03-26 14:22:05,087 : INFO : 117 batches submitted to accumulate stats from 7488 documents (23573461 virtual) 2018-03-26 14:22:13,789 : INFO : 118 batches submitted to accumulate stats from 7552 documents (23752332 virtual) 2018-03-26 14:22:19,205 : INFO : 119 batches submitted to accumulate stats from 7616 documents (23918700 virtual) 2018-03-26 14:22:26,681 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24125668 virtual) 2018-03-26 14:22:27,232 : INFO : 121 batches submitted to accumulate stats from 7744 documents (24221358 virtual) 2018-03-26 14:22:36,401 : INFO : 122 batches submitted to accumulate stats from 7808 documents (24337679 virtual) 2018-03-26 14:22:36,505 : INFO : 123 batches submitted to accumulate stats from 7872 documents (24509425 virtual) 2018-03-26 14:22:44,960 : INFO : 124 batches submitted to accumulate stats from 7936 documents (24655826 virtual) 2018-03-26 14:22:47,645 : INFO : 125 batches submitted to accumulate stats from 8000 documents (24754042 virtual) 2018-03-26 14:23:01,237 : INFO : serializing accumulator to return to master... 2018-03-26 14:23:01,244 : INFO : accumulator serialized 2018-03-26 14:23:02,080 : INFO : serializing accumulator to return to master... 2018-03-26 14:23:02,085 : INFO : accumulator serialized 2018-03-26 14:23:03,818 : INFO : serializing accumulator to return to master... 2018-03-26 14:23:03,824 : INFO : accumulator serialized 2018-03-26 14:23:04,224 : INFO : 3 accumulators retrieved from output queue 2018-03-26 14:23:04,568 : INFO : accumulated word occurrence stats for 24866310 virtual documents 2018-03-26 14:23:05,775 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_40 2018-03-26 14:23:05,823 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-26 14:23:05,983 : INFO : setting ignored attribute state to None 2018-03-26 14:23:05,984 : INFO : setting ignored attribute dispatcher to None 2018-03-26 14:23:05,984 : INFO : setting ignored attribute id2word to None 2018-03-26 14:23:05,985 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_40 2018-03-26 14:23:05,985 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_40.state 2018-03-26 14:23:06,349 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_40.state 2018-03-26 14:23:06,422 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 14:23:06,534 : INFO : 1 batches submitted to accumulate stats from 64 documents (190785 virtual) 2018-03-26 14:23:06,594 : INFO : 2 batches submitted to accumulate stats from 128 documents (350115 virtual) 2018-03-26 14:23:06,663 : INFO : 3 batches submitted to accumulate stats from 192 documents (497820 virtual) 2018-03-26 14:23:06,842 : INFO : 4 batches submitted to accumulate stats from 256 documents (793299 virtual) 2018-03-26 14:23:06,940 : INFO : 5 batches submitted to accumulate stats from 320 documents (960042 virtual) 2018-03-26 14:23:07,034 : INFO : 6 batches submitted to accumulate stats from 384 documents (1114698 virtual) 2018-03-26 14:23:15,543 : INFO : 7 batches submitted to accumulate stats from 448 documents (1262215 virtual) 2018-03-26 14:23:15,734 : INFO : 8 batches submitted to accumulate stats from 512 documents (1444627 virtual) 2018-03-26 14:23:17,804 : INFO : 9 batches submitted to accumulate stats from 576 documents (1623336 virtual) 2018-03-26 14:23:25,674 : INFO : 10 batches submitted to accumulate stats from 640 documents (1754795 virtual) 2018-03-26 14:23:27,333 : INFO : 11 batches submitted to accumulate stats from 704 documents (1926644 virtual) 2018-03-26 14:23:33,373 : INFO : 12 batches submitted to accumulate stats from 768 documents (2090525 virtual) 2018-03-26 14:23:33,951 : INFO : 13 batches submitted to accumulate stats from 832 documents (2253475 virtual) 2018-03-26 14:23:37,835 : INFO : 14 batches submitted to accumulate stats from 896 documents (2365438 virtual) 2018-03-26 14:23:41,815 : INFO : 15 batches submitted to accumulate stats from 960 documents (2501171 virtual) 2018-03-26 14:23:43,674 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2863738 virtual) 2018-03-26 14:23:47,864 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3045944 virtual) 2018-03-26 14:23:50,837 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3223729 virtual) 2018-03-26 14:23:52,779 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3447093 virtual) 2018-03-26 14:23:54,422 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3622983 virtual) 2018-03-26 14:23:59,039 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3790450 virtual) 2018-03-26 14:24:04,430 : INFO : 22 batches submitted to accumulate stats from 1408 documents (3967484 virtual) 2018-03-26 14:24:09,524 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4786150 virtual) 2018-03-26 14:24:13,323 : INFO : 24 batches submitted to accumulate stats from 1536 documents (4930064 virtual) 2018-03-26 14:24:16,843 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5145423 virtual) 2018-03-26 14:24:20,169 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5277964 virtual) 2018-03-26 14:24:22,547 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5475301 virtual) 2018-03-26 14:24:27,193 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5621134 virtual) 2018-03-26 14:24:30,345 : INFO : 29 batches submitted to accumulate stats from 1856 documents (5849865 virtual) 2018-03-26 14:24:37,414 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6049632 virtual) 2018-03-26 14:24:38,670 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6188621 virtual) 2018-03-26 14:24:47,360 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7790108 virtual) 2018-03-26 14:24:49,357 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8229243 virtual) 2018-03-26 14:24:59,913 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8385952 virtual) 2018-03-26 14:25:01,125 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8610324 virtual) 2018-03-26 14:25:05,993 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8765131 virtual) 2018-03-26 14:25:07,825 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8891328 virtual) 2018-03-26 14:25:17,062 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9073737 virtual) 2018-03-26 14:25:26,506 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9209908 virtual) 2018-03-26 14:25:29,608 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9375852 virtual) 2018-03-26 14:25:35,454 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9549872 virtual) 2018-03-26 14:25:36,379 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9698038 virtual) 2018-03-26 14:25:44,113 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9898220 virtual) 2018-03-26 14:25:45,559 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10079396 virtual) 2018-03-26 14:25:53,676 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10221180 virtual) 2018-03-26 14:25:55,907 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10360535 virtual) 2018-03-26 14:26:02,928 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10494281 virtual) 2018-03-26 14:26:07,894 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10675850 virtual) 2018-03-26 14:26:13,148 : INFO : 49 batches submitted to accumulate stats from 3136 documents (10811604 virtual) 2018-03-26 14:26:15,412 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11027041 virtual) 2018-03-26 14:26:20,957 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11169583 virtual) 2018-03-26 14:26:23,164 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11292870 virtual) 2018-03-26 14:26:28,336 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11455982 virtual) 2018-03-26 14:26:30,977 : INFO : 54 batches submitted to accumulate stats from 3456 documents (11624474 virtual) 2018-03-26 14:26:31,059 : INFO : 55 batches submitted to accumulate stats from 3520 documents (11749552 virtual) 2018-03-26 14:26:37,810 : INFO : 56 batches submitted to accumulate stats from 3584 documents (11924519 virtual) 2018-03-26 14:26:39,293 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12078907 virtual) 2018-03-26 14:26:39,999 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12251805 virtual) 2018-03-26 14:26:46,681 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12385368 virtual) 2018-03-26 14:26:47,235 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12544833 virtual) 2018-03-26 14:26:49,463 : INFO : 61 batches submitted to accumulate stats from 3904 documents (12766104 virtual) 2018-03-26 14:26:56,121 : INFO : 62 batches submitted to accumulate stats from 3968 documents (12895387 virtual) 2018-03-26 14:26:56,829 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13017139 virtual) 2018-03-26 14:26:59,437 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13161669 virtual) 2018-03-26 14:27:03,742 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13294733 virtual) 2018-03-26 14:27:06,070 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13432620 virtual) 2018-03-26 14:27:11,409 : INFO : 67 batches submitted to accumulate stats from 4288 documents (13557568 virtual) 2018-03-26 14:27:11,931 : INFO : 68 batches submitted to accumulate stats from 4352 documents (13756109 virtual) 2018-03-26 14:27:13,151 : INFO : 69 batches submitted to accumulate stats from 4416 documents (13904369 virtual) 2018-03-26 14:27:19,571 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14051803 virtual) 2018-03-26 14:27:19,920 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14256325 virtual) 2018-03-26 14:27:21,275 : INFO : 72 batches submitted to accumulate stats from 4608 documents (14399252 virtual) 2018-03-26 14:27:27,165 : INFO : 73 batches submitted to accumulate stats from 4672 documents (14616551 virtual) 2018-03-26 14:27:29,767 : INFO : 74 batches submitted to accumulate stats from 4736 documents (14794238 virtual) 2018-03-26 14:27:30,999 : INFO : 75 batches submitted to accumulate stats from 4800 documents (14955513 virtual) 2018-03-26 14:27:36,477 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15117228 virtual) 2018-03-26 14:27:38,642 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15259331 virtual) 2018-03-26 14:27:40,185 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15403712 virtual) 2018-03-26 14:27:48,395 : INFO : 79 batches submitted to accumulate stats from 5056 documents (15567794 virtual) 2018-03-26 14:27:48,885 : INFO : 80 batches submitted to accumulate stats from 5120 documents (15735327 virtual) 2018-03-26 14:27:49,507 : INFO : 81 batches submitted to accumulate stats from 5184 documents (15869280 virtual) 2018-03-26 14:27:57,083 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16008016 virtual) 2018-03-26 14:27:57,960 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16190141 virtual) 2018-03-26 14:27:58,154 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16366370 virtual) 2018-03-26 14:28:06,040 : INFO : 85 batches submitted to accumulate stats from 5440 documents (16510752 virtual) 2018-03-26 14:28:06,328 : INFO : 86 batches submitted to accumulate stats from 5504 documents (16762782 virtual) 2018-03-26 14:28:08,131 : INFO : 87 batches submitted to accumulate stats from 5568 documents (16940231 virtual) 2018-03-26 14:28:14,210 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17193051 virtual) 2018-03-26 14:28:16,196 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17312681 virtual) 2018-03-26 14:28:18,445 : INFO : 90 batches submitted to accumulate stats from 5760 documents (17454235 virtual) 2018-03-26 14:28:22,167 : INFO : 91 batches submitted to accumulate stats from 5824 documents (17600934 virtual) 2018-03-26 14:28:28,446 : INFO : 92 batches submitted to accumulate stats from 5888 documents (17769348 virtual) 2018-03-26 14:28:29,840 : INFO : 93 batches submitted to accumulate stats from 5952 documents (17941992 virtual) 2018-03-26 14:28:35,260 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18223475 virtual) 2018-03-26 14:28:35,440 : INFO : 95 batches submitted to accumulate stats from 6080 documents (18366238 virtual) 2018-03-26 14:28:37,955 : INFO : 96 batches submitted to accumulate stats from 6144 documents (18494225 virtual) 2018-03-26 14:28:44,344 : INFO : 97 batches submitted to accumulate stats from 6208 documents (18650287 virtual) 2018-03-26 14:28:45,415 : INFO : 98 batches submitted to accumulate stats from 6272 documents (18782481 virtual) 2018-03-26 14:28:48,264 : INFO : 99 batches submitted to accumulate stats from 6336 documents (18905134 virtual) 2018-03-26 14:28:53,736 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19098500 virtual) 2018-03-26 14:28:55,446 : INFO : 101 batches submitted to accumulate stats from 6464 documents (19282166 virtual) 2018-03-26 14:29:01,058 : INFO : 102 batches submitted to accumulate stats from 6528 documents (19499973 virtual) 2018-03-26 14:29:02,809 : INFO : 103 batches submitted to accumulate stats from 6592 documents (19695994 virtual) 2018-03-26 14:29:02,963 : INFO : 104 batches submitted to accumulate stats from 6656 documents (19856407 virtual) 2018-03-26 14:29:08,485 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20008305 virtual) 2018-03-26 14:29:13,606 : INFO : 106 batches submitted to accumulate stats from 6784 documents (20140226 virtual) 2018-03-26 14:29:14,505 : INFO : 107 batches submitted to accumulate stats from 6848 documents (21730454 virtual) 2018-03-26 14:29:20,051 : INFO : 108 batches submitted to accumulate stats from 6912 documents (21833943 virtual) 2018-03-26 14:29:23,241 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22017805 virtual) 2018-03-26 14:29:24,681 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22176298 virtual) 2018-03-26 14:29:29,094 : INFO : 111 batches submitted to accumulate stats from 7104 documents (22309049 virtual) 2018-03-26 14:29:30,985 : INFO : 112 batches submitted to accumulate stats from 7168 documents (22502878 virtual) 2018-03-26 14:29:34,525 : INFO : 113 batches submitted to accumulate stats from 7232 documents (22798398 virtual) 2018-03-26 14:29:40,798 : INFO : 114 batches submitted to accumulate stats from 7296 documents (22971420 virtual) 2018-03-26 14:29:43,940 : INFO : 115 batches submitted to accumulate stats from 7360 documents (23236959 virtual) 2018-03-26 14:29:48,635 : INFO : 116 batches submitted to accumulate stats from 7424 documents (23384273 virtual) 2018-03-26 14:29:52,944 : INFO : 117 batches submitted to accumulate stats from 7488 documents (23573539 virtual) 2018-03-26 14:30:02,027 : INFO : 118 batches submitted to accumulate stats from 7552 documents (23762730 virtual) 2018-03-26 14:30:05,719 : INFO : 119 batches submitted to accumulate stats from 7616 documents (23921345 virtual) 2018-03-26 14:30:13,666 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24125766 virtual) 2018-03-26 14:30:16,410 : INFO : 121 batches submitted to accumulate stats from 7744 documents (24222589 virtual) 2018-03-26 14:30:23,700 : INFO : 122 batches submitted to accumulate stats from 7808 documents (24337749 virtual) 2018-03-26 14:30:26,768 : INFO : 123 batches submitted to accumulate stats from 7872 documents (24509442 virtual) 2018-03-26 14:30:32,403 : INFO : 124 batches submitted to accumulate stats from 7936 documents (24655843 virtual) 2018-03-26 14:30:37,836 : INFO : 125 batches submitted to accumulate stats from 8000 documents (24754059 virtual) 2018-03-26 14:30:51,775 : INFO : serializing accumulator to return to master... 2018-03-26 14:30:51,821 : INFO : accumulator serialized 2018-03-26 14:30:52,529 : INFO : serializing accumulator to return to master... 2018-03-26 14:30:52,535 : INFO : accumulator serialized 2018-03-26 14:30:53,902 : INFO : serializing accumulator to return to master... 2018-03-26 14:30:53,908 : INFO : accumulator serialized 2018-03-26 14:30:54,382 : INFO : 3 accumulators retrieved from output queue 2018-03-26 14:30:54,810 : INFO : accumulated word occurrence stats for 24866310 virtual documents 2018-03-26 14:30:56,188 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_50 2018-03-26 14:30:56,245 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_50.expElogbeta.npy with mmap=None 2018-03-26 14:30:56,452 : INFO : setting ignored attribute state to None 2018-03-26 14:30:56,453 : INFO : setting ignored attribute dispatcher to None 2018-03-26 14:30:56,453 : INFO : setting ignored attribute id2word to None 2018-03-26 14:30:56,454 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_50 2018-03-26 14:30:56,454 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_50.state 2018-03-26 14:30:56,876 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_50.state 2018-03-26 14:30:56,957 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 14:30:57,072 : INFO : 1 batches submitted to accumulate stats from 64 documents (190785 virtual) 2018-03-26 14:30:57,132 : INFO : 2 batches submitted to accumulate stats from 128 documents (350115 virtual) 2018-03-26 14:30:57,210 : INFO : 3 batches submitted to accumulate stats from 192 documents (497820 virtual) 2018-03-26 14:30:57,390 : INFO : 4 batches submitted to accumulate stats from 256 documents (793299 virtual) 2018-03-26 14:30:57,494 : INFO : 5 batches submitted to accumulate stats from 320 documents (960042 virtual) 2018-03-26 14:30:57,589 : INFO : 6 batches submitted to accumulate stats from 384 documents (1109608 virtual) 2018-03-26 14:31:07,063 : INFO : 7 batches submitted to accumulate stats from 448 documents (1258779 virtual) 2018-03-26 14:31:09,728 : INFO : 8 batches submitted to accumulate stats from 512 documents (1444598 virtual) 2018-03-26 14:31:09,846 : INFO : 9 batches submitted to accumulate stats from 576 documents (1623239 virtual) 2018-03-26 14:31:19,557 : INFO : 10 batches submitted to accumulate stats from 640 documents (1754739 virtual) 2018-03-26 14:31:20,507 : INFO : 11 batches submitted to accumulate stats from 704 documents (1917819 virtual) 2018-03-26 14:31:26,822 : INFO : 12 batches submitted to accumulate stats from 768 documents (2083102 virtual) 2018-03-26 14:31:29,100 : INFO : 13 batches submitted to accumulate stats from 832 documents (2248568 virtual) 2018-03-26 14:31:32,471 : INFO : 14 batches submitted to accumulate stats from 896 documents (2363028 virtual) 2018-03-26 14:31:37,668 : INFO : 15 batches submitted to accumulate stats from 960 documents (2498649 virtual) 2018-03-26 14:31:40,603 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2858973 virtual) 2018-03-26 14:31:43,050 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3045929 virtual) 2018-03-26 14:31:48,171 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3223506 virtual) 2018-03-26 14:31:50,496 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3432839 virtual) 2018-03-26 14:31:50,962 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3618307 virtual) 2018-03-26 14:31:57,036 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3779636 virtual) 2018-03-26 14:32:02,183 : INFO : 22 batches submitted to accumulate stats from 1408 documents (3967474 virtual) 2018-03-26 14:32:08,543 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4786073 virtual) 2018-03-26 14:32:13,125 : INFO : 24 batches submitted to accumulate stats from 1536 documents (4929960 virtual) 2018-03-26 14:32:15,453 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5141515 virtual) 2018-03-26 14:32:20,615 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5277957 virtual) 2018-03-26 14:32:22,934 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5475245 virtual) 2018-03-26 14:32:27,631 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5615188 virtual) 2018-03-26 14:32:31,667 : INFO : 29 batches submitted to accumulate stats from 1856 documents (5849797 virtual) 2018-03-26 14:32:39,756 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6046861 virtual) 2018-03-26 14:32:39,967 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6185308 virtual) 2018-03-26 14:32:49,044 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7790056 virtual) 2018-03-26 14:32:52,896 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8229117 virtual) 2018-03-26 14:33:03,201 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8383777 virtual) 2018-03-26 14:33:05,631 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8610270 virtual) 2018-03-26 14:33:09,505 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8757020 virtual) 2018-03-26 14:33:12,147 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8891242 virtual) 2018-03-26 14:33:22,104 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9072388 virtual) 2018-03-26 14:33:31,274 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9209920 virtual) 2018-03-26 14:33:36,171 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9375709 virtual) 2018-03-26 14:33:40,630 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9547522 virtual) 2018-03-26 14:33:44,176 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9695401 virtual) 2018-03-26 14:33:51,572 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9893696 virtual) 2018-03-26 14:33:52,744 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10062986 virtual) 2018-03-26 14:34:03,341 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10220994 virtual) 2018-03-26 14:34:04,798 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10360338 virtual) 2018-03-26 14:34:12,915 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10486533 virtual) 2018-03-26 14:34:16,964 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10675730 virtual) 2018-03-26 14:34:23,020 : INFO : 49 batches submitted to accumulate stats from 3136 documents (10805858 virtual) 2018-03-26 14:34:26,290 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11017078 virtual) 2018-03-26 14:34:31,632 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11166450 virtual) 2018-03-26 14:34:34,475 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11292602 virtual) 2018-03-26 14:34:40,151 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11453433 virtual) 2018-03-26 14:34:42,566 : INFO : 54 batches submitted to accumulate stats from 3456 documents (11618734 virtual) 2018-03-26 14:34:42,917 : INFO : 55 batches submitted to accumulate stats from 3520 documents (11741775 virtual) 2018-03-26 14:34:51,938 : INFO : 56 batches submitted to accumulate stats from 3584 documents (11912282 virtual) 2018-03-26 14:34:52,617 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12075806 virtual) 2018-03-26 14:34:52,930 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12247887 virtual) 2018-03-26 14:35:00,845 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12381758 virtual) 2018-03-26 14:35:01,822 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12538424 virtual) 2018-03-26 14:35:03,453 : INFO : 61 batches submitted to accumulate stats from 3904 documents (12765289 virtual) 2018-03-26 14:35:11,861 : INFO : 62 batches submitted to accumulate stats from 3968 documents (12894667 virtual) 2018-03-26 14:35:12,632 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13014223 virtual) 2018-03-26 14:35:14,008 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13161686 virtual) 2018-03-26 14:35:20,424 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13294694 virtual) 2018-03-26 14:35:23,157 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13432584 virtual) 2018-03-26 14:35:28,309 : INFO : 67 batches submitted to accumulate stats from 4288 documents (13550152 virtual) 2018-03-26 14:35:29,225 : INFO : 68 batches submitted to accumulate stats from 4352 documents (13744487 virtual) 2018-03-26 14:35:31,141 : INFO : 69 batches submitted to accumulate stats from 4416 documents (13904026 virtual) 2018-03-26 14:35:37,775 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14051597 virtual) 2018-03-26 14:35:37,925 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14254450 virtual) 2018-03-26 14:35:40,246 : INFO : 72 batches submitted to accumulate stats from 4608 documents (14395258 virtual) 2018-03-26 14:35:45,653 : INFO : 73 batches submitted to accumulate stats from 4672 documents (14612925 virtual) 2018-03-26 14:35:50,303 : INFO : 74 batches submitted to accumulate stats from 4736 documents (14791128 virtual) 2018-03-26 14:35:50,421 : INFO : 75 batches submitted to accumulate stats from 4800 documents (14944261 virtual) 2018-03-26 14:35:55,390 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15107449 virtual) 2018-03-26 14:35:58,715 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15254117 virtual) 2018-03-26 14:36:02,035 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15396967 virtual) 2018-03-26 14:36:09,276 : INFO : 79 batches submitted to accumulate stats from 5056 documents (15561868 virtual) 2018-03-26 14:36:09,386 : INFO : 80 batches submitted to accumulate stats from 5120 documents (15734998 virtual) 2018-03-26 14:36:12,032 : INFO : 81 batches submitted to accumulate stats from 5184 documents (15869098 virtual) 2018-03-26 14:36:18,721 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16007936 virtual) 2018-03-26 14:36:20,312 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16183885 virtual) 2018-03-26 14:36:21,652 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16366326 virtual) 2018-03-26 14:36:29,106 : INFO : 85 batches submitted to accumulate stats from 5440 documents (16510713 virtual) 2018-03-26 14:36:30,330 : INFO : 86 batches submitted to accumulate stats from 5504 documents (16759507 virtual) 2018-03-26 14:36:31,933 : INFO : 87 batches submitted to accumulate stats from 5568 documents (16935783 virtual) 2018-03-26 14:36:37,882 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17177816 virtual) 2018-03-26 14:36:41,521 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17306754 virtual) 2018-03-26 14:36:43,755 : INFO : 90 batches submitted to accumulate stats from 5760 documents (17448534 virtual) 2018-03-26 14:36:47,651 : INFO : 91 batches submitted to accumulate stats from 5824 documents (17600817 virtual) 2018-03-26 14:36:54,731 : INFO : 92 batches submitted to accumulate stats from 5888 documents (17769228 virtual) 2018-03-26 14:36:56,144 : INFO : 93 batches submitted to accumulate stats from 5952 documents (17938088 virtual) 2018-03-26 14:37:01,297 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18222297 virtual) 2018-03-26 14:37:03,292 : INFO : 95 batches submitted to accumulate stats from 6080 documents (18361878 virtual) 2018-03-26 14:37:05,165 : INFO : 96 batches submitted to accumulate stats from 6144 documents (18489708 virtual) 2018-03-26 14:37:11,211 : INFO : 97 batches submitted to accumulate stats from 6208 documents (18639966 virtual) 2018-03-26 14:37:13,785 : INFO : 98 batches submitted to accumulate stats from 6272 documents (18782415 virtual) 2018-03-26 14:37:16,046 : INFO : 99 batches submitted to accumulate stats from 6336 documents (18904992 virtual) 2018-03-26 14:37:22,978 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19093345 virtual) 2018-03-26 14:37:24,071 : INFO : 101 batches submitted to accumulate stats from 6464 documents (19282059 virtual) 2018-03-26 14:37:29,460 : INFO : 102 batches submitted to accumulate stats from 6528 documents (19499870 virtual) 2018-03-26 14:37:33,068 : INFO : 103 batches submitted to accumulate stats from 6592 documents (19695834 virtual) 2018-03-26 14:37:33,176 : INFO : 104 batches submitted to accumulate stats from 6656 documents (19856224 virtual) 2018-03-26 14:37:37,671 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20008251 virtual) 2018-03-26 14:37:44,655 : INFO : 106 batches submitted to accumulate stats from 6784 documents (20140213 virtual) 2018-03-26 14:37:45,619 : INFO : 107 batches submitted to accumulate stats from 6848 documents (21730369 virtual) 2018-03-26 14:37:50,869 : INFO : 108 batches submitted to accumulate stats from 6912 documents (21833870 virtual) 2018-03-26 14:37:55,671 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22008821 virtual) 2018-03-26 14:37:56,642 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22175970 virtual) 2018-03-26 14:38:00,555 : INFO : 111 batches submitted to accumulate stats from 7104 documents (22308604 virtual) 2018-03-26 14:38:04,463 : INFO : 112 batches submitted to accumulate stats from 7168 documents (22502517 virtual) 2018-03-26 14:38:06,636 : INFO : 113 batches submitted to accumulate stats from 7232 documents (22791633 virtual) 2018-03-26 14:38:14,688 : INFO : 114 batches submitted to accumulate stats from 7296 documents (22971100 virtual) 2018-03-26 14:38:18,018 : INFO : 115 batches submitted to accumulate stats from 7360 documents (23221550 virtual) 2018-03-26 14:38:23,469 : INFO : 116 batches submitted to accumulate stats from 7424 documents (23370985 virtual) 2018-03-26 14:38:27,972 : INFO : 117 batches submitted to accumulate stats from 7488 documents (23560677 virtual) 2018-03-26 14:38:38,648 : INFO : 118 batches submitted to accumulate stats from 7552 documents (23752298 virtual) 2018-03-26 14:38:41,461 : INFO : 119 batches submitted to accumulate stats from 7616 documents (23918657 virtual) 2018-03-26 14:38:50,173 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24122878 virtual) 2018-03-26 14:38:53,630 : INFO : 121 batches submitted to accumulate stats from 7744 documents (24221374 virtual) 2018-03-26 14:39:02,245 : INFO : 122 batches submitted to accumulate stats from 7808 documents (24337695 virtual) 2018-03-26 14:39:04,924 : INFO : 123 batches submitted to accumulate stats from 7872 documents (24506429 virtual) 2018-03-26 14:39:12,272 : INFO : 124 batches submitted to accumulate stats from 7936 documents (24641748 virtual) 2018-03-26 14:39:18,303 : INFO : 125 batches submitted to accumulate stats from 8000 documents (24753874 virtual) 2018-03-26 14:39:30,964 : INFO : serializing accumulator to return to master... 2018-03-26 14:39:31,050 : INFO : accumulator serialized 2018-03-26 14:39:35,605 : INFO : serializing accumulator to return to master... 2018-03-26 14:39:35,613 : INFO : accumulator serialized 2018-03-26 14:39:37,270 : INFO : serializing accumulator to return to master... 2018-03-26 14:39:37,278 : INFO : accumulator serialized 2018-03-26 14:39:37,877 : INFO : 3 accumulators retrieved from output queue 2018-03-26 14:39:38,435 : INFO : accumulated word occurrence stats for 24866312 virtual documents
In [30]:
tmfull_score['u_mass'] = tmfull_score.apply(lambda row: get_score(row.num_topic, dict_tmfull, texts=text_tmfull, corpus=corpus_tmfull,
coherence='u_mass', model_dir='../src/tmfull'),
axis=1)
2018-03-26 15:56:13,014 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_10 2018-03-26 15:56:13,099 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-26 15:56:13,157 : INFO : setting ignored attribute state to None 2018-03-26 15:56:13,158 : INFO : setting ignored attribute dispatcher to None 2018-03-26 15:56:13,158 : INFO : setting ignored attribute id2word to None 2018-03-26 15:56:13,159 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_10 2018-03-26 15:56:13,159 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_10.state 2018-03-26 15:56:13,268 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_10.state 2018-03-26 15:56:13,342 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-26 15:56:13,359 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-26 15:56:13,377 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-26 15:56:13,394 : INFO : CorpusAccumulator accumulated stats from 4000 documents 2018-03-26 15:56:13,411 : INFO : CorpusAccumulator accumulated stats from 5000 documents 2018-03-26 15:56:13,429 : INFO : CorpusAccumulator accumulated stats from 6000 documents 2018-03-26 15:56:13,447 : INFO : CorpusAccumulator accumulated stats from 7000 documents 2018-03-26 15:56:13,464 : INFO : CorpusAccumulator accumulated stats from 8000 documents 2018-03-26 15:56:13,490 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_10 2018-03-26 15:56:13,494 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-26 15:56:13,497 : INFO : setting ignored attribute state to None 2018-03-26 15:56:13,497 : INFO : setting ignored attribute dispatcher to None 2018-03-26 15:56:13,498 : INFO : setting ignored attribute id2word to None 2018-03-26 15:56:13,498 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_10 2018-03-26 15:56:13,499 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_10.state 2018-03-26 15:56:13,531 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_10.state 2018-03-26 15:56:13,573 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-26 15:56:13,591 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-26 15:56:13,608 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-26 15:56:13,625 : INFO : CorpusAccumulator accumulated stats from 4000 documents 2018-03-26 15:56:13,642 : INFO : CorpusAccumulator accumulated stats from 5000 documents 2018-03-26 15:56:13,660 : INFO : CorpusAccumulator accumulated stats from 6000 documents 2018-03-26 15:56:13,678 : INFO : CorpusAccumulator accumulated stats from 7000 documents 2018-03-26 15:56:13,695 : INFO : CorpusAccumulator accumulated stats from 8000 documents
--------------------------------------------------------------------------- ZeroDivisionError Traceback (most recent call last) <ipython-input-30-20edf4198d97> in <module>() 1 tmfull_score['u_mass'] = tmfull_score.apply(lambda row: get_score(row.num_topic, dict_tmfull, texts=text_tmfull, corpus=corpus_tmfull, 2 coherence='u_mass', model_dir='../src/tmfull'), ----> 3 axis=1) ~/miniconda3/envs/infnet3/lib/python3.6/site-packages/pandas/core/frame.py in apply(self, func, axis, broadcast, raw, reduce, args, **kwds) 4875 f, axis, 4876 reduce=reduce, -> 4877 ignore_failures=ignore_failures) 4878 else: 4879 return self._apply_broadcast(f, axis) ~/miniconda3/envs/infnet3/lib/python3.6/site-packages/pandas/core/frame.py in _apply_standard(self, func, axis, ignore_failures, reduce) 4971 try: 4972 for i, v in enumerate(series_gen): -> 4973 results[i] = func(v) 4974 keys.append(v.name) 4975 except Exception as e: <ipython-input-30-20edf4198d97> in <lambda>(row) 1 tmfull_score['u_mass'] = tmfull_score.apply(lambda row: get_score(row.num_topic, dict_tmfull, texts=text_tmfull, corpus=corpus_tmfull, ----> 2 coherence='u_mass', model_dir='../src/tmfull'), 3 axis=1) <ipython-input-8-8cbdbb8a7ac8> in get_score(num_topic, dictionary, corpus, texts, coherence, model_dir) 7 # topn was 20 by default in the original function 8 toptopics = _model.top_topics( ----> 9 corpus=corpus, texts=texts, coherence=coherence, dictionary=dictionary, topn=15) 10 avg = sum(t[1] for t in toptopics) / num_topic # Return the average measure 11 return avg ~/miniconda3/envs/infnet3/lib/python3.6/site-packages/gensim/models/ldamodel.py in top_topics(self, corpus, texts, dictionary, window_size, coherence, topn, processes) 896 processes=processes 897 ) --> 898 coherence_scores = cm.get_coherence_per_topic() 899 900 str_topics = [] ~/miniconda3/envs/infnet3/lib/python3.6/site-packages/gensim/models/coherencemodel.py in get_coherence_per_topic(self, segmented_topics, with_std, with_support) 422 kwargs['normalize'] = (self.coherence == 'c_npmi') 423 --> 424 return measure.conf(segmented_topics, self._accumulator, **kwargs) 425 426 def aggregate_measures(self, topic_coherences): ~/miniconda3/envs/infnet3/lib/python3.6/site-packages/gensim/topic_coherence/direct_confirmation_measure.py in log_conditional_probability(segmented_topics, accumulator, with_std, with_support) 45 w_star_count = accumulator[w_star] 46 co_occur_count = accumulator[w_prime, w_star] ---> 47 m_lc_i = np.log(((co_occur_count / num_docs) + EPSILON) / (w_star_count / num_docs)) 48 except KeyError: 49 m_lc_i = 0.0 ZeroDivisionError: ('float division by zero', 'occurred at index 0')
In [26]:
tmfull_score['c_npmi'] = tmfull_score.apply(lambda row: get_score(row.num_topic, dict_tmfull, texts=text_tmfull, corpus=corpus_tmfull,
coherence='c_npmi', model_dir='../src/tmfull'),
axis=1)
2018-03-26 15:13:25,902 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_10 2018-03-26 15:13:26,005 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-26 15:13:26,177 : INFO : setting ignored attribute state to None 2018-03-26 15:13:26,178 : INFO : setting ignored attribute dispatcher to None 2018-03-26 15:13:26,178 : INFO : setting ignored attribute id2word to None 2018-03-26 15:13:26,179 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_10 2018-03-26 15:13:26,179 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_10.state 2018-03-26 15:13:26,298 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_10.state 2018-03-26 15:13:26,363 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 15:13:26,470 : INFO : 1 batches submitted to accumulate stats from 64 documents (205745 virtual) 2018-03-26 15:13:26,525 : INFO : 2 batches submitted to accumulate stats from 128 documents (382316 virtual) 2018-03-26 15:13:26,590 : INFO : 3 batches submitted to accumulate stats from 192 documents (531745 virtual) 2018-03-26 15:13:26,751 : INFO : 4 batches submitted to accumulate stats from 256 documents (840148 virtual) 2018-03-26 15:13:26,845 : INFO : 5 batches submitted to accumulate stats from 320 documents (1025019 virtual) 2018-03-26 15:13:26,914 : INFO : 6 batches submitted to accumulate stats from 384 documents (1153899 virtual) 2018-03-26 15:13:29,544 : INFO : 7 batches submitted to accumulate stats from 448 documents (1327715 virtual) 2018-03-26 15:13:29,795 : INFO : 8 batches submitted to accumulate stats from 512 documents (1524767 virtual) 2018-03-26 15:13:30,522 : INFO : 9 batches submitted to accumulate stats from 576 documents (1709293 virtual) 2018-03-26 15:13:32,933 : INFO : 10 batches submitted to accumulate stats from 640 documents (1844925 virtual) 2018-03-26 15:13:33,266 : INFO : 11 batches submitted to accumulate stats from 704 documents (2014482 virtual) 2018-03-26 15:13:35,398 : INFO : 12 batches submitted to accumulate stats from 768 documents (2213113 virtual) 2018-03-26 15:13:36,110 : INFO : 13 batches submitted to accumulate stats from 832 documents (2372103 virtual) 2018-03-26 15:13:37,012 : INFO : 14 batches submitted to accumulate stats from 896 documents (2476002 virtual) 2018-03-26 15:13:38,756 : INFO : 15 batches submitted to accumulate stats from 960 documents (2723818 virtual) 2018-03-26 15:13:39,195 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3008674 virtual) 2018-03-26 15:13:40,201 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3209891 virtual) 2018-03-26 15:13:42,074 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3399182 virtual) 2018-03-26 15:13:42,207 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3638271 virtual) 2018-03-26 15:13:42,705 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3830552 virtual) 2018-03-26 15:13:46,583 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3970032 virtual) 2018-03-26 15:13:47,058 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4163782 virtual) 2018-03-26 15:13:47,536 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4969321 virtual) 2018-03-26 15:13:50,167 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5111576 virtual) 2018-03-26 15:13:51,163 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5353532 virtual) 2018-03-26 15:13:51,671 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5503645 virtual) 2018-03-26 15:13:52,741 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5708912 virtual) 2018-03-26 15:13:54,832 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5867249 virtual) 2018-03-26 15:13:55,428 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6171445 virtual) 2018-03-26 15:13:58,222 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6316898 virtual) 2018-03-26 15:13:59,415 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6446678 virtual) 2018-03-26 15:14:03,064 : INFO : 32 batches submitted to accumulate stats from 2048 documents (8074268 virtual) 2018-03-26 15:14:03,293 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8529651 virtual) 2018-03-26 15:14:06,226 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8769862 virtual) 2018-03-26 15:14:08,012 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8917752 virtual) 2018-03-26 15:14:08,362 : INFO : 36 batches submitted to accumulate stats from 2304 documents (9065584 virtual) 2018-03-26 15:14:08,929 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9247231 virtual) 2018-03-26 15:14:13,501 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9395695 virtual) 2018-03-26 15:14:15,833 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9568729 virtual) 2018-03-26 15:14:16,213 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9766589 virtual) 2018-03-26 15:14:18,729 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9906640 virtual) 2018-03-26 15:14:19,557 : INFO : 42 batches submitted to accumulate stats from 2688 documents (10116354 virtual) 2018-03-26 15:14:21,508 : INFO : 43 batches submitted to accumulate stats from 2752 documents (10272905 virtual) 2018-03-26 15:14:22,696 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10461556 virtual) 2018-03-26 15:14:25,201 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10620144 virtual) 2018-03-26 15:14:25,450 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10751135 virtual) 2018-03-26 15:14:28,150 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10940485 virtual) 2018-03-26 15:14:29,221 : INFO : 48 batches submitted to accumulate stats from 3072 documents (11108959 virtual) 2018-03-26 15:14:31,666 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11310566 virtual) 2018-03-26 15:14:32,312 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11476827 virtual) 2018-03-26 15:14:34,190 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11613637 virtual) 2018-03-26 15:14:35,949 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11772221 virtual) 2018-03-26 15:14:37,350 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11963629 virtual) 2018-03-26 15:14:39,553 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12093173 virtual) 2018-03-26 15:14:39,941 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12282107 virtual) 2018-03-26 15:14:40,667 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12442912 virtual) 2018-03-26 15:14:42,201 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12624133 virtual) 2018-03-26 15:14:42,808 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12768910 virtual) 2018-03-26 15:14:44,240 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12944936 virtual) 2018-03-26 15:14:44,695 : INFO : 60 batches submitted to accumulate stats from 3840 documents (13172549 virtual) 2018-03-26 15:14:46,506 : INFO : 61 batches submitted to accumulate stats from 3904 documents (13311908 virtual) 2018-03-26 15:14:47,300 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13446629 virtual) 2018-03-26 15:14:47,989 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13595319 virtual) 2018-03-26 15:14:49,281 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13736889 virtual) 2018-03-26 15:14:50,514 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13886373 virtual) 2018-03-26 15:14:52,024 : INFO : 66 batches submitted to accumulate stats from 4224 documents (14057576 virtual) 2018-03-26 15:14:52,366 : INFO : 67 batches submitted to accumulate stats from 4288 documents (14234755 virtual) 2018-03-26 15:14:53,007 : INFO : 68 batches submitted to accumulate stats from 4352 documents (14405106 virtual) 2018-03-26 15:14:54,865 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14533770 virtual) 2018-03-26 15:14:55,109 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14785998 virtual) 2018-03-26 15:14:56,000 : INFO : 71 batches submitted to accumulate stats from 4544 documents (15008017 virtual) 2018-03-26 15:14:58,273 : INFO : 72 batches submitted to accumulate stats from 4608 documents (15165684 virtual) 2018-03-26 15:14:58,369 : INFO : 73 batches submitted to accumulate stats from 4672 documents (15339083 virtual) 2018-03-26 15:14:58,957 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15528101 virtual) 2018-03-26 15:15:00,910 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15688896 virtual) 2018-03-26 15:15:02,933 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15829510 virtual) 2018-03-26 15:15:03,050 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15986959 virtual) 2018-03-26 15:15:03,952 : INFO : 78 batches submitted to accumulate stats from 4992 documents (16198071 virtual) 2018-03-26 15:15:06,114 : INFO : 79 batches submitted to accumulate stats from 5056 documents (16322769 virtual) 2018-03-26 15:15:06,793 : INFO : 80 batches submitted to accumulate stats from 5120 documents (16497897 virtual) 2018-03-26 15:15:07,076 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16660878 virtual) 2018-03-26 15:15:08,943 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16869728 virtual) 2018-03-26 15:15:09,814 : INFO : 83 batches submitted to accumulate stats from 5312 documents (17002029 virtual) 2018-03-26 15:15:11,018 : INFO : 84 batches submitted to accumulate stats from 5376 documents (17294721 virtual) 2018-03-26 15:15:11,335 : INFO : 85 batches submitted to accumulate stats from 5440 documents (17458212 virtual) 2018-03-26 15:15:13,053 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17707135 virtual) 2018-03-26 15:15:14,285 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17864077 virtual) 2018-03-26 15:15:15,403 : INFO : 88 batches submitted to accumulate stats from 5632 documents (18005836 virtual) 2018-03-26 15:15:15,487 : INFO : 89 batches submitted to accumulate stats from 5696 documents (18172814 virtual) 2018-03-26 15:15:18,412 : INFO : 90 batches submitted to accumulate stats from 5760 documents (18355589 virtual) 2018-03-26 15:15:19,251 : INFO : 91 batches submitted to accumulate stats from 5824 documents (18531676 virtual) 2018-03-26 15:15:20,234 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18810862 virtual) 2018-03-26 15:15:21,447 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18964531 virtual) 2018-03-26 15:15:22,114 : INFO : 94 batches submitted to accumulate stats from 6016 documents (19102801 virtual) 2018-03-26 15:15:23,357 : INFO : 95 batches submitted to accumulate stats from 6080 documents (19274255 virtual) 2018-03-26 15:15:25,070 : INFO : 96 batches submitted to accumulate stats from 6144 documents (19408478 virtual) 2018-03-26 15:15:25,540 : INFO : 97 batches submitted to accumulate stats from 6208 documents (19538049 virtual) 2018-03-26 15:15:28,056 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19741342 virtual) 2018-03-26 15:15:28,158 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19939037 virtual) 2018-03-26 15:15:28,621 : INFO : 100 batches submitted to accumulate stats from 6400 documents (20177663 virtual) 2018-03-26 15:15:30,718 : INFO : 101 batches submitted to accumulate stats from 6464 documents (20378731 virtual) 2018-03-26 15:15:31,086 : INFO : 102 batches submitted to accumulate stats from 6528 documents (20523755 virtual) 2018-03-26 15:15:31,447 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20699276 virtual) 2018-03-26 15:15:34,665 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20841879 virtual) 2018-03-26 15:15:35,466 : INFO : 105 batches submitted to accumulate stats from 6720 documents (22429558 virtual) 2018-03-26 15:15:36,242 : INFO : 106 batches submitted to accumulate stats from 6784 documents (22585757 virtual) 2018-03-26 15:15:37,819 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22754049 virtual) 2018-03-26 15:15:38,735 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22905743 virtual) 2018-03-26 15:15:39,503 : INFO : 109 batches submitted to accumulate stats from 6976 documents (23060741 virtual) 2018-03-26 15:15:40,629 : INFO : 110 batches submitted to accumulate stats from 7040 documents (23266613 virtual) 2018-03-26 15:15:42,499 : INFO : 111 batches submitted to accumulate stats from 7104 documents (23589025 virtual) 2018-03-26 15:15:43,959 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23738308 virtual) 2018-03-26 15:15:45,161 : INFO : 113 batches submitted to accumulate stats from 7232 documents (24030149 virtual) 2018-03-26 15:15:46,887 : INFO : 114 batches submitted to accumulate stats from 7296 documents (24201786 virtual) 2018-03-26 15:15:48,526 : INFO : 115 batches submitted to accumulate stats from 7360 documents (24393107 virtual) 2018-03-26 15:15:51,306 : INFO : 116 batches submitted to accumulate stats from 7424 documents (24581823 virtual) 2018-03-26 15:15:52,894 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24761103 virtual) 2018-03-26 15:15:56,084 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24964643 virtual) 2018-03-26 15:15:56,679 : INFO : 119 batches submitted to accumulate stats from 7616 documents (25050729 virtual) 2018-03-26 15:15:59,816 : INFO : 120 batches submitted to accumulate stats from 7680 documents (25201726 virtual) 2018-03-26 15:16:00,283 : INFO : 121 batches submitted to accumulate stats from 7744 documents (25373821 virtual) 2018-03-26 15:16:03,095 : INFO : 122 batches submitted to accumulate stats from 7808 documents (25548267 virtual) 2018-03-26 15:16:04,294 : INFO : 123 batches submitted to accumulate stats from 7872 documents (25549770 virtual) 2018-03-26 15:16:08,017 : INFO : serializing accumulator to return to master... 2018-03-26 15:16:08,365 : INFO : accumulator serialized 2018-03-26 15:16:09,006 : INFO : serializing accumulator to return to master... 2018-03-26 15:16:09,011 : INFO : accumulator serialized 2018-03-26 15:16:09,624 : INFO : serializing accumulator to return to master... 2018-03-26 15:16:09,630 : INFO : accumulator serialized 2018-03-26 15:16:09,747 : INFO : 3 accumulators retrieved from output queue 2018-03-26 15:16:09,788 : INFO : accumulated word occurrence stats for 25549991 virtual documents 2018-03-26 15:16:10,136 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_15 2018-03-26 15:16:10,190 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_15.expElogbeta.npy with mmap=None 2018-03-26 15:16:10,277 : INFO : setting ignored attribute state to None 2018-03-26 15:16:10,278 : INFO : setting ignored attribute dispatcher to None 2018-03-26 15:16:10,279 : INFO : setting ignored attribute id2word to None 2018-03-26 15:16:10,279 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_15 2018-03-26 15:16:10,280 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_15.state 2018-03-26 15:16:10,421 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_15.state 2018-03-26 15:16:10,466 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 15:16:10,580 : INFO : 1 batches submitted to accumulate stats from 64 documents (197181 virtual) 2018-03-26 15:16:10,638 : INFO : 2 batches submitted to accumulate stats from 128 documents (371824 virtual) 2018-03-26 15:16:10,699 : INFO : 3 batches submitted to accumulate stats from 192 documents (520698 virtual) 2018-03-26 15:16:10,870 : INFO : 4 batches submitted to accumulate stats from 256 documents (831326 virtual) 2018-03-26 15:16:10,947 : INFO : 5 batches submitted to accumulate stats from 320 documents (993683 virtual) 2018-03-26 15:16:11,033 : INFO : 6 batches submitted to accumulate stats from 384 documents (1153316 virtual) 2018-03-26 15:16:14,038 : INFO : 7 batches submitted to accumulate stats from 448 documents (1320992 virtual) 2018-03-26 15:16:14,200 : INFO : 8 batches submitted to accumulate stats from 512 documents (1508960 virtual) 2018-03-26 15:16:14,695 : INFO : 9 batches submitted to accumulate stats from 576 documents (1695982 virtual) 2018-03-26 15:16:17,372 : INFO : 10 batches submitted to accumulate stats from 640 documents (1833108 virtual) 2018-03-26 15:16:18,036 : INFO : 11 batches submitted to accumulate stats from 704 documents (2008256 virtual) 2018-03-26 15:16:20,609 : INFO : 12 batches submitted to accumulate stats from 768 documents (2184809 virtual) 2018-03-26 15:16:20,809 : INFO : 13 batches submitted to accumulate stats from 832 documents (2366452 virtual) 2018-03-26 15:16:21,898 : INFO : 14 batches submitted to accumulate stats from 896 documents (2466651 virtual) 2018-03-26 15:16:23,599 : INFO : 15 batches submitted to accumulate stats from 960 documents (2653376 virtual) 2018-03-26 15:16:24,512 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2998011 virtual) 2018-03-26 15:16:25,707 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3187743 virtual) 2018-03-26 15:16:27,389 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3390841 virtual) 2018-03-26 15:16:27,787 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3609527 virtual) 2018-03-26 15:16:28,103 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3786504 virtual) 2018-03-26 15:16:31,478 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3952233 virtual) 2018-03-26 15:16:32,013 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4143812 virtual) 2018-03-26 15:16:34,992 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4957376 virtual) 2018-03-26 15:16:35,628 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5092919 virtual) 2018-03-26 15:16:36,714 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5330714 virtual) 2018-03-26 15:16:38,528 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5473366 virtual) 2018-03-26 15:16:39,272 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5680177 virtual) 2018-03-26 15:16:40,622 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5837768 virtual) 2018-03-26 15:16:42,158 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6083432 virtual) 2018-03-26 15:16:45,490 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6277636 virtual) 2018-03-26 15:16:45,682 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6416413 virtual) 2018-03-26 15:16:49,210 : INFO : 32 batches submitted to accumulate stats from 2048 documents (8023118 virtual) 2018-03-26 15:16:49,988 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8480544 virtual) 2018-03-26 15:16:53,937 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8719608 virtual) 2018-03-26 15:16:54,076 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8873322 virtual) 2018-03-26 15:16:55,757 : INFO : 36 batches submitted to accumulate stats from 2304 documents (9025766 virtual) 2018-03-26 15:16:56,663 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9176980 virtual) 2018-03-26 15:17:01,518 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9353748 virtual) 2018-03-26 15:17:03,936 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9523820 virtual) 2018-03-26 15:17:04,551 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9720846 virtual) 2018-03-26 15:17:07,164 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9869752 virtual) 2018-03-26 15:17:07,661 : INFO : 42 batches submitted to accumulate stats from 2688 documents (10041438 virtual) 2018-03-26 15:17:10,768 : INFO : 43 batches submitted to accumulate stats from 2752 documents (10220531 virtual) 2018-03-26 15:17:11,080 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10410056 virtual) 2018-03-26 15:17:14,235 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10578271 virtual) 2018-03-26 15:17:14,907 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10708563 virtual) 2018-03-26 15:17:17,638 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10874519 virtual) 2018-03-26 15:17:18,465 : INFO : 48 batches submitted to accumulate stats from 3072 documents (11039513 virtual) 2018-03-26 15:17:21,759 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11230763 virtual) 2018-03-26 15:17:21,963 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11425799 virtual) 2018-03-26 15:17:24,472 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11554062 virtual) 2018-03-26 15:17:25,364 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11671614 virtual) 2018-03-26 15:17:26,475 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11881915 virtual) 2018-03-26 15:17:27,885 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12027218 virtual) 2018-03-26 15:17:29,303 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12202313 virtual) 2018-03-26 15:17:30,578 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12364436 virtual) 2018-03-26 15:17:30,667 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12542893 virtual) 2018-03-26 15:17:31,656 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12720670 virtual) 2018-03-26 15:17:33,713 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12859552 virtual) 2018-03-26 15:17:34,924 : INFO : 60 batches submitted to accumulate stats from 3840 documents (13055166 virtual) 2018-03-26 15:17:35,241 : INFO : 61 batches submitted to accumulate stats from 3904 documents (13231658 virtual) 2018-03-26 15:17:37,156 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13363326 virtual) 2018-03-26 15:17:38,641 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13532971 virtual) 2018-03-26 15:17:38,894 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13661154 virtual) 2018-03-26 15:17:40,005 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13801528 virtual) 2018-03-26 15:17:42,522 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13936423 virtual) 2018-03-26 15:17:42,628 : INFO : 67 batches submitted to accumulate stats from 4288 documents (14129736 virtual) 2018-03-26 15:17:42,760 : INFO : 68 batches submitted to accumulate stats from 4352 documents (14311306 virtual) 2018-03-26 15:17:45,189 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14479396 virtual) 2018-03-26 15:17:45,756 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14680738 virtual) 2018-03-26 15:17:45,939 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14829475 virtual) 2018-03-26 15:17:48,113 : INFO : 72 batches submitted to accumulate stats from 4608 documents (15064423 virtual) 2018-03-26 15:17:49,476 : INFO : 73 batches submitted to accumulate stats from 4672 documents (15230712 virtual) 2018-03-26 15:17:49,860 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15398969 virtual) 2018-03-26 15:17:51,621 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15579091 virtual) 2018-03-26 15:17:52,912 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15733186 virtual) 2018-03-26 15:17:53,567 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15885165 virtual) 2018-03-26 15:17:56,216 : INFO : 78 batches submitted to accumulate stats from 4992 documents (16050292 virtual) 2018-03-26 15:17:56,419 : INFO : 79 batches submitted to accumulate stats from 5056 documents (16235914 virtual) 2018-03-26 15:17:57,063 : INFO : 80 batches submitted to accumulate stats from 5120 documents (16369682 virtual) 2018-03-26 15:17:59,554 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16532356 virtual) 2018-03-26 15:18:00,032 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16714705 virtual) 2018-03-26 15:18:00,466 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16903546 virtual) 2018-03-26 15:18:03,027 : INFO : 84 batches submitted to accumulate stats from 5376 documents (17054394 virtual) 2018-03-26 15:18:03,164 : INFO : 85 batches submitted to accumulate stats from 5440 documents (17315733 virtual) 2018-03-26 15:18:04,013 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17567381 virtual) 2018-03-26 15:18:06,310 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17757116 virtual) 2018-03-26 15:18:07,003 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17898251 virtual) 2018-03-26 15:18:07,999 : INFO : 89 batches submitted to accumulate stats from 5696 documents (18035526 virtual) 2018-03-26 15:18:09,388 : INFO : 90 batches submitted to accumulate stats from 5760 documents (18183357 virtual) 2018-03-26 15:18:11,601 : INFO : 91 batches submitted to accumulate stats from 5824 documents (18384081 virtual) 2018-03-26 15:18:13,247 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18559585 virtual) 2018-03-26 15:18:13,472 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18855926 virtual) 2018-03-26 15:18:14,604 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18992268 virtual) 2018-03-26 15:18:16,241 : INFO : 95 batches submitted to accumulate stats from 6080 documents (19128024 virtual) 2018-03-26 15:18:16,533 : INFO : 96 batches submitted to accumulate stats from 6144 documents (19301225 virtual) 2018-03-26 15:18:18,719 : INFO : 97 batches submitted to accumulate stats from 6208 documents (19434891 virtual) 2018-03-26 15:18:19,818 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19557220 virtual) 2018-03-26 15:18:21,567 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19792961 virtual) 2018-03-26 15:18:22,536 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19977468 virtual) 2018-03-26 15:18:22,777 : INFO : 101 batches submitted to accumulate stats from 6464 documents (20186353 virtual) 2018-03-26 15:18:25,223 : INFO : 102 batches submitted to accumulate stats from 6528 documents (20401203 virtual) 2018-03-26 15:18:25,383 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20555148 virtual) 2018-03-26 15:18:25,473 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20716487 virtual) 2018-03-26 15:18:29,021 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20852427 virtual) 2018-03-26 15:18:29,957 : INFO : 106 batches submitted to accumulate stats from 6784 documents (22436848 virtual) 2018-03-26 15:18:30,433 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22587138 virtual) 2018-03-26 15:18:33,218 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22764968 virtual) 2018-03-26 15:18:33,642 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22910771 virtual) 2018-03-26 15:18:33,743 : INFO : 110 batches submitted to accumulate stats from 7040 documents (23061822 virtual) 2018-03-26 15:18:36,030 : INFO : 111 batches submitted to accumulate stats from 7104 documents (23267808 virtual) 2018-03-26 15:18:36,932 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23600451 virtual) 2018-03-26 15:18:39,727 : INFO : 113 batches submitted to accumulate stats from 7232 documents (23739337 virtual) 2018-03-26 15:18:39,879 : INFO : 114 batches submitted to accumulate stats from 7296 documents (24031088 virtual) 2018-03-26 15:18:42,874 : INFO : 115 batches submitted to accumulate stats from 7360 documents (24198969 virtual) 2018-03-26 15:18:43,426 : INFO : 116 batches submitted to accumulate stats from 7424 documents (24383937 virtual) 2018-03-26 15:18:46,350 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24580667 virtual) 2018-03-26 15:18:49,526 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24745891 virtual) 2018-03-26 15:18:52,073 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24948411 virtual) 2018-03-26 15:18:53,022 : INFO : 120 batches submitted to accumulate stats from 7680 documents (25047798 virtual) 2018-03-26 15:18:55,944 : INFO : 121 batches submitted to accumulate stats from 7744 documents (25193344 virtual) 2018-03-26 15:18:56,978 : INFO : 122 batches submitted to accumulate stats from 7808 documents (25367434 virtual) 2018-03-26 15:18:59,485 : INFO : 123 batches submitted to accumulate stats from 7872 documents (25525632 virtual) 2018-03-26 15:19:00,922 : INFO : 124 batches submitted to accumulate stats from 7936 documents (25550875 virtual) 2018-03-26 15:19:05,603 : INFO : serializing accumulator to return to master... 2018-03-26 15:19:05,610 : INFO : accumulator serialized 2018-03-26 15:19:05,842 : INFO : serializing accumulator to return to master... 2018-03-26 15:19:05,847 : INFO : accumulator serialized 2018-03-26 15:19:06,666 : INFO : serializing accumulator to return to master... 2018-03-26 15:19:06,671 : INFO : accumulator serialized 2018-03-26 15:19:06,815 : INFO : 3 accumulators retrieved from output queue 2018-03-26 15:19:06,912 : INFO : accumulated word occurrence stats for 25551129 virtual documents 2018-03-26 15:19:07,388 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_20 2018-03-26 15:19:07,676 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_20.expElogbeta.npy with mmap=None 2018-03-26 15:19:07,759 : INFO : setting ignored attribute state to None 2018-03-26 15:19:07,760 : INFO : setting ignored attribute dispatcher to None 2018-03-26 15:19:07,760 : INFO : setting ignored attribute id2word to None 2018-03-26 15:19:07,761 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_20 2018-03-26 15:19:07,761 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_20.state 2018-03-26 15:19:07,937 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_20.state 2018-03-26 15:19:07,994 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 15:19:08,106 : INFO : 1 batches submitted to accumulate stats from 64 documents (197185 virtual) 2018-03-26 15:19:08,161 : INFO : 2 batches submitted to accumulate stats from 128 documents (362915 virtual) 2018-03-26 15:19:08,231 : INFO : 3 batches submitted to accumulate stats from 192 documents (517020 virtual) 2018-03-26 15:19:08,410 : INFO : 4 batches submitted to accumulate stats from 256 documents (818899 virtual) 2018-03-26 15:19:08,521 : INFO : 5 batches submitted to accumulate stats from 320 documents (992042 virtual) 2018-03-26 15:19:08,614 : INFO : 6 batches submitted to accumulate stats from 384 documents (1153098 virtual) 2018-03-26 15:19:11,762 : INFO : 7 batches submitted to accumulate stats from 448 documents (1307015 virtual) 2018-03-26 15:19:11,865 : INFO : 8 batches submitted to accumulate stats from 512 documents (1495827 virtual) 2018-03-26 15:19:12,567 : INFO : 9 batches submitted to accumulate stats from 576 documents (1680936 virtual) 2018-03-26 15:19:15,496 : INFO : 10 batches submitted to accumulate stats from 640 documents (1818795 virtual) 2018-03-26 15:19:16,097 : INFO : 11 batches submitted to accumulate stats from 704 documents (2000866 virtual) 2018-03-26 15:19:18,382 : INFO : 12 batches submitted to accumulate stats from 768 documents (2171486 virtual) 2018-03-26 15:19:18,764 : INFO : 13 batches submitted to accumulate stats from 832 documents (2337797 virtual) 2018-03-26 15:19:20,491 : INFO : 14 batches submitted to accumulate stats from 896 documents (2457918 virtual) 2018-03-26 15:19:22,356 : INFO : 15 batches submitted to accumulate stats from 960 documents (2599989 virtual) 2018-03-26 15:19:23,075 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2972836 virtual) 2018-03-26 15:19:25,066 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3154858 virtual) 2018-03-26 15:19:26,500 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3339062 virtual) 2018-03-26 15:19:26,901 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3569027 virtual) 2018-03-26 15:19:27,708 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3755563 virtual) 2018-03-26 15:19:29,670 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3925090 virtual) 2018-03-26 15:19:31,766 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4109915 virtual) 2018-03-26 15:19:33,728 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4938588 virtual) 2018-03-26 15:19:35,096 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5084129 virtual) 2018-03-26 15:19:36,876 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5307230 virtual) 2018-03-26 15:19:37,829 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5447532 virtual) 2018-03-26 15:19:38,691 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5659758 virtual) 2018-03-26 15:19:40,858 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5815410 virtual) 2018-03-26 15:19:41,579 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6048242 virtual) 2018-03-26 15:19:44,665 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6252659 virtual) 2018-03-26 15:19:45,751 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6403564 virtual) 2018-03-26 15:19:49,213 : INFO : 32 batches submitted to accumulate stats from 2048 documents (8006647 virtual) 2018-03-26 15:19:49,635 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8443004 virtual) 2018-03-26 15:19:53,970 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8621759 virtual) 2018-03-26 15:19:54,193 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8859278 virtual) 2018-03-26 15:19:55,830 : INFO : 36 batches submitted to accumulate stats from 2304 documents (9012192 virtual) 2018-03-26 15:19:57,228 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9145901 virtual) 2018-03-26 15:20:01,174 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9337382 virtual) 2018-03-26 15:20:04,645 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9488807 virtual) 2018-03-26 15:20:06,306 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9657314 virtual) 2018-03-26 15:20:08,175 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9828004 virtual) 2018-03-26 15:20:09,218 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9990733 virtual) 2018-03-26 15:20:12,261 : INFO : 43 batches submitted to accumulate stats from 2752 documents (10207744 virtual) 2018-03-26 15:20:12,605 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10384097 virtual) 2018-03-26 15:20:16,055 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10548164 virtual) 2018-03-26 15:20:16,314 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10697370 virtual) 2018-03-26 15:20:19,645 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10844108 virtual) 2018-03-26 15:20:20,790 : INFO : 48 batches submitted to accumulate stats from 3072 documents (11005488 virtual) 2018-03-26 15:20:23,510 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11166308 virtual) 2018-03-26 15:20:24,367 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11382859 virtual) 2018-03-26 15:20:26,868 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11508417 virtual) 2018-03-26 15:20:27,569 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11640444 virtual) 2018-03-26 15:20:27,676 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11843725 virtual) 2018-03-26 15:20:30,203 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12001289 virtual) 2018-03-26 15:20:30,997 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12161422 virtual) 2018-03-26 15:20:32,450 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12336949 virtual) 2018-03-26 15:20:33,031 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12499054 virtual) 2018-03-26 15:20:33,955 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12680189 virtual) 2018-03-26 15:20:36,594 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12807576 virtual) 2018-03-26 15:20:36,701 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12997357 virtual) 2018-03-26 15:20:37,512 : INFO : 61 batches submitted to accumulate stats from 3904 documents (13198413 virtual) 2018-03-26 15:20:40,290 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13331875 virtual) 2018-03-26 15:20:40,445 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13481361 virtual) 2018-03-26 15:20:41,407 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13617558 virtual) 2018-03-26 15:20:43,123 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13751490 virtual) 2018-03-26 15:20:44,466 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13901323 virtual) 2018-03-26 15:20:45,961 : INFO : 67 batches submitted to accumulate stats from 4288 documents (14073296 virtual) 2018-03-26 15:20:46,095 : INFO : 68 batches submitted to accumulate stats from 4352 documents (14237466 virtual) 2018-03-26 15:20:47,705 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14420388 virtual) 2018-03-26 15:20:48,907 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14537142 virtual) 2018-03-26 15:20:49,111 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14790947 virtual) 2018-03-26 15:20:51,019 : INFO : 72 batches submitted to accumulate stats from 4608 documents (15009178 virtual) 2018-03-26 15:20:52,523 : INFO : 73 batches submitted to accumulate stats from 4672 documents (15166839 virtual) 2018-03-26 15:20:52,900 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15340159 virtual) 2018-03-26 15:20:55,055 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15529173 virtual) 2018-03-26 15:20:55,141 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15682942 virtual) 2018-03-26 15:20:58,209 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15816972 virtual) 2018-03-26 15:20:58,520 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15967399 virtual) 2018-03-26 15:20:59,748 : INFO : 79 batches submitted to accumulate stats from 5056 documents (16181905 virtual) 2018-03-26 15:21:01,968 : INFO : 80 batches submitted to accumulate stats from 5120 documents (16299008 virtual) 2018-03-26 15:21:02,745 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16457542 virtual) 2018-03-26 15:21:03,303 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16622026 virtual) 2018-03-26 15:21:04,827 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16832894 virtual) 2018-03-26 15:21:06,084 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16966756 virtual) 2018-03-26 15:21:07,288 : INFO : 85 batches submitted to accumulate stats from 5440 documents (17255197 virtual) 2018-03-26 15:21:08,063 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17428635 virtual) 2018-03-26 15:21:09,707 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17680292 virtual) 2018-03-26 15:21:11,075 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17825909 virtual) 2018-03-26 15:21:12,477 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17955148 virtual) 2018-03-26 15:21:12,738 : INFO : 90 batches submitted to accumulate stats from 5760 documents (18112255 virtual) 2018-03-26 15:21:16,315 : INFO : 91 batches submitted to accumulate stats from 5824 documents (18305177 virtual) 2018-03-26 15:21:16,739 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18464232 virtual) 2018-03-26 15:21:18,178 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18640293 virtual) 2018-03-26 15:21:19,479 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18912293 virtual) 2018-03-26 15:21:19,648 : INFO : 95 batches submitted to accumulate stats from 6080 documents (19043497 virtual) 2018-03-26 15:21:21,675 : INFO : 96 batches submitted to accumulate stats from 6144 documents (19196459 virtual) 2018-03-26 15:21:23,109 : INFO : 97 batches submitted to accumulate stats from 6208 documents (19378518 virtual) 2018-03-26 15:21:23,762 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19496403 virtual) 2018-03-26 15:21:25,439 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19655295 virtual) 2018-03-26 15:21:26,702 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19857642 virtual) 2018-03-26 15:21:28,632 : INFO : 101 batches submitted to accumulate stats from 6464 documents (20072684 virtual) 2018-03-26 15:21:29,146 : INFO : 102 batches submitted to accumulate stats from 6528 documents (20298313 virtual) 2018-03-26 15:21:30,569 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20468689 virtual) 2018-03-26 15:21:31,373 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20651029 virtual) 2018-03-26 15:21:32,865 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20795144 virtual) 2018-03-26 15:21:35,200 : INFO : 106 batches submitted to accumulate stats from 6784 documents (22363377 virtual) 2018-03-26 15:21:36,291 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22485996 virtual) 2018-03-26 15:21:37,956 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22657475 virtual) 2018-03-26 15:21:38,849 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22827094 virtual) 2018-03-26 15:21:40,460 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22980624 virtual) 2018-03-26 15:21:41,184 : INFO : 111 batches submitted to accumulate stats from 7104 documents (23189774 virtual) 2018-03-26 15:21:43,247 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23458525 virtual) 2018-03-26 15:21:44,730 : INFO : 113 batches submitted to accumulate stats from 7232 documents (23678204 virtual) 2018-03-26 15:21:46,986 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23926483 virtual) 2018-03-26 15:21:48,102 : INFO : 115 batches submitted to accumulate stats from 7360 documents (24087817 virtual) 2018-03-26 15:21:50,848 : INFO : 116 batches submitted to accumulate stats from 7424 documents (24274628 virtual) 2018-03-26 15:21:53,931 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24443802 virtual) 2018-03-26 15:21:55,729 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24643750 virtual) 2018-03-26 15:21:58,917 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24833799 virtual) 2018-03-26 15:21:59,392 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24988366 virtual) 2018-03-26 15:22:03,296 : INFO : 121 batches submitted to accumulate stats from 7744 documents (25077991 virtual) 2018-03-26 15:22:03,455 : INFO : 122 batches submitted to accumulate stats from 7808 documents (25248223 virtual) 2018-03-26 15:22:07,569 : INFO : 123 batches submitted to accumulate stats from 7872 documents (25421880 virtual) 2018-03-26 15:22:07,868 : INFO : 124 batches submitted to accumulate stats from 7936 documents (25551237 virtual) 2018-03-26 15:22:13,547 : INFO : serializing accumulator to return to master... 2018-03-26 15:22:13,553 : INFO : accumulator serialized 2018-03-26 15:22:14,522 : INFO : serializing accumulator to return to master... 2018-03-26 15:22:14,528 : INFO : accumulator serialized 2018-03-26 15:22:14,851 : INFO : serializing accumulator to return to master... 2018-03-26 15:22:14,855 : INFO : accumulator serialized 2018-03-26 15:22:15,036 : INFO : 3 accumulators retrieved from output queue 2018-03-26 15:22:15,165 : INFO : accumulated word occurrence stats for 25551537 virtual documents 2018-03-26 15:22:15,795 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_25 2018-03-26 15:22:15,845 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-26 15:22:15,953 : INFO : setting ignored attribute state to None 2018-03-26 15:22:15,953 : INFO : setting ignored attribute dispatcher to None 2018-03-26 15:22:15,954 : INFO : setting ignored attribute id2word to None 2018-03-26 15:22:15,954 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_25 2018-03-26 15:22:15,955 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_25.state 2018-03-26 15:22:16,169 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_25.state 2018-03-26 15:22:16,226 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 15:22:16,341 : INFO : 1 batches submitted to accumulate stats from 64 documents (197179 virtual) 2018-03-26 15:22:16,398 : INFO : 2 batches submitted to accumulate stats from 128 documents (362960 virtual) 2018-03-26 15:22:16,468 : INFO : 3 batches submitted to accumulate stats from 192 documents (520587 virtual) 2018-03-26 15:22:16,656 : INFO : 4 batches submitted to accumulate stats from 256 documents (822479 virtual) 2018-03-26 15:22:16,758 : INFO : 5 batches submitted to accumulate stats from 320 documents (993646 virtual) 2018-03-26 15:22:16,856 : INFO : 6 batches submitted to accumulate stats from 384 documents (1153215 virtual) 2018-03-26 15:22:19,995 : INFO : 7 batches submitted to accumulate stats from 448 documents (1307109 virtual) 2018-03-26 15:22:20,239 : INFO : 8 batches submitted to accumulate stats from 512 documents (1507697 virtual) 2018-03-26 15:22:21,241 : INFO : 9 batches submitted to accumulate stats from 576 documents (1682589 virtual) 2018-03-26 15:22:24,105 : INFO : 10 batches submitted to accumulate stats from 640 documents (1822128 virtual) 2018-03-26 15:22:24,975 : INFO : 11 batches submitted to accumulate stats from 704 documents (2007365 virtual) 2018-03-26 15:22:26,953 : INFO : 12 batches submitted to accumulate stats from 768 documents (2171596 virtual) 2018-03-26 15:22:27,571 : INFO : 13 batches submitted to accumulate stats from 832 documents (2345217 virtual) 2018-03-26 15:22:29,669 : INFO : 14 batches submitted to accumulate stats from 896 documents (2458210 virtual) 2018-03-26 15:22:30,827 : INFO : 15 batches submitted to accumulate stats from 960 documents (2620061 virtual) 2018-03-26 15:22:31,018 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2974369 virtual) 2018-03-26 15:22:34,127 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3166619 virtual) 2018-03-26 15:22:34,703 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3351861 virtual) 2018-03-26 15:22:34,829 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3574250 virtual) 2018-03-26 15:22:36,778 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3761825 virtual) 2018-03-26 15:22:38,540 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3932167 virtual) 2018-03-26 15:22:41,106 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4120818 virtual) 2018-03-26 15:22:42,800 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4939731 virtual) 2018-03-26 15:22:42,913 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5085782 virtual) 2018-03-26 15:22:46,236 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5318482 virtual) 2018-03-26 15:22:46,834 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5447489 virtual) 2018-03-26 15:22:47,169 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5663066 virtual) 2018-03-26 15:22:50,420 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5815377 virtual) 2018-03-26 15:22:50,554 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6048226 virtual) 2018-03-26 15:22:53,358 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6252716 virtual) 2018-03-26 15:22:55,602 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6406543 virtual) 2018-03-26 15:22:58,665 : INFO : 32 batches submitted to accumulate stats from 2048 documents (8011906 virtual) 2018-03-26 15:22:59,329 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8451297 virtual) 2018-03-26 15:23:03,857 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8693702 virtual) 2018-03-26 15:23:04,109 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8859367 virtual) 2018-03-26 15:23:05,330 : INFO : 36 batches submitted to accumulate stats from 2304 documents (9012292 virtual) 2018-03-26 15:23:07,330 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9145938 virtual) 2018-03-26 15:23:12,815 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9340253 virtual) 2018-03-26 15:23:14,066 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9488854 virtual) 2018-03-26 15:23:16,524 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9661822 virtual) 2018-03-26 15:23:17,719 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9828037 virtual) 2018-03-26 15:23:19,459 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9990705 virtual) 2018-03-26 15:23:22,190 : INFO : 43 batches submitted to accumulate stats from 2752 documents (10207716 virtual) 2018-03-26 15:23:22,713 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10384069 virtual) 2018-03-26 15:23:26,110 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10548972 virtual) 2018-03-26 15:23:26,699 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10697445 virtual) 2018-03-26 15:23:29,886 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10844290 virtual) 2018-03-26 15:23:31,430 : INFO : 48 batches submitted to accumulate stats from 3072 documents (11005526 virtual) 2018-03-26 15:23:33,854 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11180984 virtual) 2018-03-26 15:23:35,134 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11394123 virtual) 2018-03-26 15:23:37,344 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11516756 virtual) 2018-03-26 15:23:38,489 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11640485 virtual) 2018-03-26 15:23:39,684 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11843699 virtual) 2018-03-26 15:23:40,994 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12001264 virtual) 2018-03-26 15:23:42,596 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12161397 virtual) 2018-03-26 15:23:43,840 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12336924 virtual) 2018-03-26 15:23:44,409 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12499029 virtual) 2018-03-26 15:23:45,511 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12679012 virtual) 2018-03-26 15:23:48,059 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12807543 virtual) 2018-03-26 15:23:48,363 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12997324 virtual) 2018-03-26 15:23:49,370 : INFO : 61 batches submitted to accumulate stats from 3904 documents (13198288 virtual) 2018-03-26 15:23:51,995 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13326988 virtual) 2018-03-26 15:23:52,083 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13473205 virtual) 2018-03-26 15:23:53,445 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13617515 virtual) 2018-03-26 15:23:55,051 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13751447 virtual) 2018-03-26 15:23:56,194 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13901176 virtual) 2018-03-26 15:23:58,030 : INFO : 67 batches submitted to accumulate stats from 4288 documents (14064198 virtual) 2018-03-26 15:23:58,213 : INFO : 68 batches submitted to accumulate stats from 4352 documents (14237353 virtual) 2018-03-26 15:23:59,610 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14418686 virtual) 2018-03-26 15:24:01,278 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14534833 virtual) 2018-03-26 15:24:01,415 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14786991 virtual) 2018-03-26 15:24:03,078 : INFO : 72 batches submitted to accumulate stats from 4608 documents (15008971 virtual) 2018-03-26 15:24:05,323 : INFO : 73 batches submitted to accumulate stats from 4672 documents (15166558 virtual) 2018-03-26 15:24:05,427 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15339775 virtual) 2018-03-26 15:24:06,917 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15520924 virtual) 2018-03-26 15:24:08,100 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15679380 virtual) 2018-03-26 15:24:11,104 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15803513 virtual) 2018-03-26 15:24:11,726 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15951268 virtual) 2018-03-26 15:24:11,860 : INFO : 79 batches submitted to accumulate stats from 5056 documents (16172642 virtual) 2018-03-26 15:24:15,062 : INFO : 80 batches submitted to accumulate stats from 5120 documents (16287120 virtual) 2018-03-26 15:24:15,432 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16437440 virtual) 2018-03-26 15:24:16,110 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16616702 virtual) 2018-03-26 15:24:17,835 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16818775 virtual) 2018-03-26 15:24:18,791 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16963671 virtual) 2018-03-26 15:24:20,504 : INFO : 85 batches submitted to accumulate stats from 5440 documents (17242931 virtual) 2018-03-26 15:24:21,319 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17417468 virtual) 2018-03-26 15:24:22,269 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17668640 virtual) 2018-03-26 15:24:24,796 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17820744 virtual) 2018-03-26 15:24:25,538 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17945520 virtual) 2018-03-26 15:24:25,940 : INFO : 90 batches submitted to accumulate stats from 5760 documents (18107053 virtual) 2018-03-26 15:24:29,441 : INFO : 91 batches submitted to accumulate stats from 5824 documents (18297035 virtual) 2018-03-26 15:24:30,428 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18445578 virtual) 2018-03-26 15:24:31,676 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18621574 virtual) 2018-03-26 15:24:32,827 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18906839 virtual) 2018-03-26 15:24:33,459 : INFO : 95 batches submitted to accumulate stats from 6080 documents (19043087 virtual) 2018-03-26 15:24:35,510 : INFO : 96 batches submitted to accumulate stats from 6144 documents (19182601 virtual) 2018-03-26 15:24:36,796 : INFO : 97 batches submitted to accumulate stats from 6208 documents (19367218 virtual) 2018-03-26 15:24:37,313 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19486199 virtual) 2018-03-26 15:24:39,485 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19649529 virtual) 2018-03-26 15:24:40,652 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19856105 virtual) 2018-03-26 15:24:43,305 : INFO : 101 batches submitted to accumulate stats from 6464 documents (20061390 virtual) 2018-03-26 15:24:44,128 : INFO : 102 batches submitted to accumulate stats from 6528 documents (20288794 virtual) 2018-03-26 15:24:45,414 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20462255 virtual) 2018-03-26 15:24:46,198 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20600023 virtual) 2018-03-26 15:24:48,066 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20778699 virtual) 2018-03-26 15:24:50,330 : INFO : 106 batches submitted to accumulate stats from 6784 documents (22350807 virtual) 2018-03-26 15:24:51,078 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22474875 virtual) 2018-03-26 15:24:53,472 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22642691 virtual) 2018-03-26 15:24:54,265 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22821796 virtual) 2018-03-26 15:24:54,453 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22964175 virtual) 2018-03-26 15:24:57,261 : INFO : 111 batches submitted to accumulate stats from 7104 documents (23171069 virtual) 2018-03-26 15:24:57,639 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23417327 virtual) 2018-03-26 15:25:00,973 : INFO : 113 batches submitted to accumulate stats from 7232 documents (23675779 virtual) 2018-03-26 15:25:01,778 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23921307 virtual) 2018-03-26 15:25:04,202 : INFO : 115 batches submitted to accumulate stats from 7360 documents (24076034 virtual) 2018-03-26 15:25:05,729 : INFO : 116 batches submitted to accumulate stats from 7424 documents (24265981 virtual) 2018-03-26 15:25:09,781 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24427019 virtual) 2018-03-26 15:25:11,468 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24612536 virtual) 2018-03-26 15:25:14,971 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24824739 virtual) 2018-03-26 15:25:15,059 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24978134 virtual) 2018-03-26 15:25:18,575 : INFO : 121 batches submitted to accumulate stats from 7744 documents (25069581 virtual) 2018-03-26 15:25:19,468 : INFO : 122 batches submitted to accumulate stats from 7808 documents (25243626 virtual) 2018-03-26 15:25:22,617 : INFO : 123 batches submitted to accumulate stats from 7872 documents (25408411 virtual) 2018-03-26 15:25:24,401 : INFO : 124 batches submitted to accumulate stats from 7936 documents (25551102 virtual) 2018-03-26 15:25:30,114 : INFO : serializing accumulator to return to master... 2018-03-26 15:25:30,225 : INFO : serializing accumulator to return to master... 2018-03-26 15:25:30,121 : INFO : accumulator serialized 2018-03-26 15:25:30,234 : INFO : accumulator serialized 2018-03-26 15:25:32,146 : INFO : serializing accumulator to return to master... 2018-03-26 15:25:32,152 : INFO : accumulator serialized 2018-03-26 15:25:32,381 : INFO : 3 accumulators retrieved from output queue 2018-03-26 15:25:32,553 : INFO : accumulated word occurrence stats for 25551422 virtual documents 2018-03-26 15:25:33,301 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_30 2018-03-26 15:25:33,385 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-26 15:25:33,517 : INFO : setting ignored attribute state to None 2018-03-26 15:25:33,517 : INFO : setting ignored attribute dispatcher to None 2018-03-26 15:25:33,518 : INFO : setting ignored attribute id2word to None 2018-03-26 15:25:33,518 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_30 2018-03-26 15:25:33,519 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_30.state 2018-03-26 15:25:33,784 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_30.state 2018-03-26 15:25:33,864 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 15:25:33,979 : INFO : 1 batches submitted to accumulate stats from 64 documents (197185 virtual) 2018-03-26 15:25:34,036 : INFO : 2 batches submitted to accumulate stats from 128 documents (362915 virtual) 2018-03-26 15:25:34,109 : INFO : 3 batches submitted to accumulate stats from 192 documents (517020 virtual) 2018-03-26 15:25:34,312 : INFO : 4 batches submitted to accumulate stats from 256 documents (818899 virtual) 2018-03-26 15:25:34,423 : INFO : 5 batches submitted to accumulate stats from 320 documents (992042 virtual) 2018-03-26 15:25:34,514 : INFO : 6 batches submitted to accumulate stats from 384 documents (1148008 virtual) 2018-03-26 15:25:37,908 : INFO : 7 batches submitted to accumulate stats from 448 documents (1303579 virtual) 2018-03-26 15:25:38,016 : INFO : 8 batches submitted to accumulate stats from 512 documents (1495798 virtual) 2018-03-26 15:25:38,723 : INFO : 9 batches submitted to accumulate stats from 576 documents (1680839 virtual) 2018-03-26 15:25:42,095 : INFO : 10 batches submitted to accumulate stats from 640 documents (1818739 virtual) 2018-03-26 15:25:42,418 : INFO : 11 batches submitted to accumulate stats from 704 documents (1988219 virtual) 2018-03-26 15:25:45,211 : INFO : 12 batches submitted to accumulate stats from 768 documents (2159902 virtual) 2018-03-26 15:25:45,714 : INFO : 13 batches submitted to accumulate stats from 832 documents (2331768 virtual) 2018-03-26 15:25:46,997 : INFO : 14 batches submitted to accumulate stats from 896 documents (2452628 virtual) 2018-03-26 15:25:49,162 : INFO : 15 batches submitted to accumulate stats from 960 documents (2594649 virtual) 2018-03-26 15:25:49,577 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2961373 virtual) 2018-03-26 15:25:51,102 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3154729 virtual) 2018-03-26 15:25:53,352 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3338822 virtual) 2018-03-26 15:25:53,617 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3558812 virtual) 2018-03-26 15:25:53,959 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3750984 virtual) 2018-03-26 15:25:56,805 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3924851 virtual) 2018-03-26 15:25:58,499 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4108285 virtual) 2018-03-26 15:26:01,221 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4933351 virtual) 2018-03-26 15:26:02,620 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5083665 virtual) 2018-03-26 15:26:03,784 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5305424 virtual) 2018-03-26 15:26:06,030 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5447406 virtual) 2018-03-26 15:26:06,917 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5655665 virtual) 2018-03-26 15:26:08,256 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5807741 virtual) 2018-03-26 15:26:10,437 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6037478 virtual) 2018-03-26 15:26:13,503 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6246053 virtual) 2018-03-26 15:26:13,876 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6389035 virtual) 2018-03-26 15:26:17,704 : INFO : 32 batches submitted to accumulate stats from 2048 documents (8002267 virtual) 2018-03-26 15:26:18,596 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8442646 virtual) 2018-03-26 15:26:22,729 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8603771 virtual) 2018-03-26 15:26:23,613 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8836730 virtual) 2018-03-26 15:26:25,159 : INFO : 36 batches submitted to accumulate stats from 2304 documents (9003935 virtual) 2018-03-26 15:26:25,884 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9138828 virtual) 2018-03-26 15:26:29,776 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9321929 virtual) 2018-03-26 15:26:34,972 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9475585 virtual) 2018-03-26 15:26:35,255 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9643615 virtual) 2018-03-26 15:26:38,309 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9816097 virtual) 2018-03-26 15:26:38,835 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9970432 virtual) 2018-03-26 15:26:42,347 : INFO : 43 batches submitted to accumulate stats from 2752 documents (10190788 virtual) 2018-03-26 15:26:42,510 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10370095 virtual) 2018-03-26 15:26:46,464 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10519712 virtual) 2018-03-26 15:26:46,641 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10671468 virtual) 2018-03-26 15:26:50,197 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10817922 virtual) 2018-03-26 15:26:51,650 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10990924 virtual) 2018-03-26 15:26:54,492 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11141883 virtual) 2018-03-26 15:26:55,179 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11355126 virtual) 2018-03-26 15:26:57,954 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11505130 virtual) 2018-03-26 15:26:58,738 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11634694 virtual) 2018-03-26 15:27:00,185 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11820262 virtual) 2018-03-26 15:27:02,028 : INFO : 54 batches submitted to accumulate stats from 3456 documents (11976029 virtual) 2018-03-26 15:27:02,256 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12112701 virtual) 2018-03-26 15:27:05,346 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12298101 virtual) 2018-03-26 15:27:05,442 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12461305 virtual) 2018-03-26 15:27:05,600 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12628558 virtual) 2018-03-26 15:27:08,685 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12774447 virtual) 2018-03-26 15:27:09,298 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12952770 virtual) 2018-03-26 15:27:09,635 : INFO : 61 batches submitted to accumulate stats from 3904 documents (13173432 virtual) 2018-03-26 15:27:13,003 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13309540 virtual) 2018-03-26 15:27:13,379 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13441684 virtual) 2018-03-26 15:27:13,468 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13579868 virtual) 2018-03-26 15:27:16,497 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13725381 virtual) 2018-03-26 15:27:17,557 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13873209 virtual) 2018-03-26 15:27:18,771 : INFO : 67 batches submitted to accumulate stats from 4288 documents (14022676 virtual) 2018-03-26 15:27:19,821 : INFO : 68 batches submitted to accumulate stats from 4352 documents (14219812 virtual) 2018-03-26 15:27:20,805 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14374187 virtual) 2018-03-26 15:27:22,086 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14510690 virtual) 2018-03-26 15:27:23,534 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14734426 virtual) 2018-03-26 15:27:24,595 : INFO : 72 batches submitted to accumulate stats from 4608 documents (14879275 virtual) 2018-03-26 15:27:26,033 : INFO : 73 batches submitted to accumulate stats from 4672 documents (15111523 virtual) 2018-03-26 15:27:28,288 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15295307 virtual) 2018-03-26 15:27:28,424 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15473227 virtual) 2018-03-26 15:27:29,413 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15651393 virtual) 2018-03-26 15:27:31,941 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15772253 virtual) 2018-03-26 15:27:33,320 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15924745 virtual) 2018-03-26 15:27:34,788 : INFO : 79 batches submitted to accumulate stats from 5056 documents (16112931 virtual) 2018-03-26 15:27:36,069 : INFO : 80 batches submitted to accumulate stats from 5120 documents (16266553 virtual) 2018-03-26 15:27:37,828 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16405217 virtual) 2018-03-26 15:27:38,875 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16561315 virtual) 2018-03-26 15:27:38,988 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16755275 virtual) 2018-03-26 15:27:41,740 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16919947 virtual) 2018-03-26 15:27:42,535 : INFO : 85 batches submitted to accumulate stats from 5440 documents (17086906 virtual) 2018-03-26 15:27:43,411 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17360505 virtual) 2018-03-26 15:27:45,149 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17599096 virtual) 2018-03-26 15:27:46,390 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17788873 virtual) 2018-03-26 15:27:47,924 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17917615 virtual) 2018-03-26 15:27:49,103 : INFO : 90 batches submitted to accumulate stats from 5760 documents (18063268 virtual) 2018-03-26 15:27:50,341 : INFO : 91 batches submitted to accumulate stats from 5824 documents (18209246 virtual) 2018-03-26 15:27:53,947 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18409455 virtual) 2018-03-26 15:27:54,628 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18569430 virtual) 2018-03-26 15:27:54,952 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18883302 virtual) 2018-03-26 15:27:57,090 : INFO : 95 batches submitted to accumulate stats from 6080 documents (19003681 virtual) 2018-03-26 15:27:58,227 : INFO : 96 batches submitted to accumulate stats from 6144 documents (19131533 virtual) 2018-03-26 15:27:58,350 : INFO : 97 batches submitted to accumulate stats from 6208 documents (19321572 virtual) 2018-03-26 15:28:01,884 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19443973 virtual) 2018-03-26 15:28:02,043 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19559209 virtual) 2018-03-26 15:28:04,864 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19797936 virtual) 2018-03-26 15:28:05,048 : INFO : 101 batches submitted to accumulate stats from 6464 documents (19995287 virtual) 2018-03-26 15:28:05,717 : INFO : 102 batches submitted to accumulate stats from 6528 documents (20189959 virtual) 2018-03-26 15:28:07,961 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20401764 virtual) 2018-03-26 15:28:08,462 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20555563 virtual) 2018-03-26 15:28:09,435 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20716935 virtual) 2018-03-26 15:28:13,206 : INFO : 106 batches submitted to accumulate stats from 6784 documents (20852741 virtual) 2018-03-26 15:28:14,092 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22437221 virtual) 2018-03-26 15:28:14,196 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22587459 virtual) 2018-03-26 15:28:17,552 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22765289 virtual) 2018-03-26 15:28:18,089 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22911092 virtual) 2018-03-26 15:28:18,497 : INFO : 111 batches submitted to accumulate stats from 7104 documents (23062085 virtual) 2018-03-26 15:28:20,912 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23267860 virtual) 2018-03-26 15:28:22,113 : INFO : 113 batches submitted to accumulate stats from 7232 documents (23581689 virtual) 2018-03-26 15:28:25,232 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23732154 virtual) 2018-03-26 15:28:25,548 : INFO : 115 batches submitted to accumulate stats from 7360 documents (24016952 virtual) 2018-03-26 15:28:28,781 : INFO : 116 batches submitted to accumulate stats from 7424 documents (24189458 virtual) 2018-03-26 15:28:29,732 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24362328 virtual) 2018-03-26 15:28:33,117 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24575119 virtual) 2018-03-26 15:28:36,165 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24711086 virtual) 2018-03-26 15:28:39,614 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24936253 virtual) 2018-03-26 15:28:40,361 : INFO : 121 batches submitted to accumulate stats from 7744 documents (25034620 virtual) 2018-03-26 15:28:43,734 : INFO : 122 batches submitted to accumulate stats from 7808 documents (25170156 virtual) 2018-03-26 15:28:45,266 : INFO : 123 batches submitted to accumulate stats from 7872 documents (25330650 virtual) 2018-03-26 15:28:46,928 : INFO : 124 batches submitted to accumulate stats from 7936 documents (25512587 virtual) 2018-03-26 15:28:49,192 : INFO : 125 batches submitted to accumulate stats from 8000 documents (25551256 virtual) 2018-03-26 15:28:54,504 : INFO : serializing accumulator to return to master... 2018-03-26 15:28:54,469 : INFO : serializing accumulator to return to master... 2018-03-26 15:28:54,565 : INFO : accumulator serialized 2018-03-26 15:28:54,566 : INFO : accumulator serialized 2018-03-26 15:28:55,969 : INFO : serializing accumulator to return to master... 2018-03-26 15:28:55,975 : INFO : accumulator serialized 2018-03-26 15:28:56,255 : INFO : 3 accumulators retrieved from output queue 2018-03-26 15:28:56,513 : INFO : accumulated word occurrence stats for 25551601 virtual documents 2018-03-26 15:28:57,430 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_35 2018-03-26 15:28:57,490 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_35.expElogbeta.npy with mmap=None 2018-03-26 15:28:57,639 : INFO : setting ignored attribute state to None 2018-03-26 15:28:57,640 : INFO : setting ignored attribute dispatcher to None 2018-03-26 15:28:57,640 : INFO : setting ignored attribute id2word to None 2018-03-26 15:28:57,641 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_35 2018-03-26 15:28:57,641 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_35.state 2018-03-26 15:28:57,943 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_35.state 2018-03-26 15:28:58,010 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 15:28:58,127 : INFO : 1 batches submitted to accumulate stats from 64 documents (197185 virtual) 2018-03-26 15:28:58,186 : INFO : 2 batches submitted to accumulate stats from 128 documents (362915 virtual) 2018-03-26 15:28:58,261 : INFO : 3 batches submitted to accumulate stats from 192 documents (517020 virtual) 2018-03-26 15:28:58,444 : INFO : 4 batches submitted to accumulate stats from 256 documents (818899 virtual) 2018-03-26 15:28:58,550 : INFO : 5 batches submitted to accumulate stats from 320 documents (992042 virtual) 2018-03-26 15:28:58,645 : INFO : 6 batches submitted to accumulate stats from 384 documents (1148008 virtual) 2018-03-26 15:29:02,219 : INFO : 7 batches submitted to accumulate stats from 448 documents (1303579 virtual) 2018-03-26 15:29:02,334 : INFO : 8 batches submitted to accumulate stats from 512 documents (1495798 virtual) 2018-03-26 15:29:03,163 : INFO : 9 batches submitted to accumulate stats from 576 documents (1680839 virtual) 2018-03-26 15:29:06,657 : INFO : 10 batches submitted to accumulate stats from 640 documents (1818739 virtual) 2018-03-26 15:29:06,909 : INFO : 11 batches submitted to accumulate stats from 704 documents (1988219 virtual) 2018-03-26 15:29:09,536 : INFO : 12 batches submitted to accumulate stats from 768 documents (2159902 virtual) 2018-03-26 15:29:10,486 : INFO : 13 batches submitted to accumulate stats from 832 documents (2331768 virtual) 2018-03-26 15:29:11,541 : INFO : 14 batches submitted to accumulate stats from 896 documents (2452628 virtual) 2018-03-26 15:29:13,874 : INFO : 15 batches submitted to accumulate stats from 960 documents (2594649 virtual) 2018-03-26 15:29:14,226 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2961373 virtual) 2018-03-26 15:29:15,800 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3154729 virtual) 2018-03-26 15:29:18,145 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3338706 virtual) 2018-03-26 15:29:18,351 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3554439 virtual) 2018-03-26 15:29:18,725 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3746211 virtual) 2018-03-26 15:29:21,747 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3911575 virtual) 2018-03-26 15:29:23,458 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4103718 virtual) 2018-03-26 15:29:26,281 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4929572 virtual) 2018-03-26 15:29:27,363 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5074664 virtual) 2018-03-26 15:29:28,870 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5290593 virtual) 2018-03-26 15:29:31,060 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5444346 virtual) 2018-03-26 15:29:31,318 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5648034 virtual) 2018-03-26 15:29:33,663 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5794377 virtual) 2018-03-26 15:29:34,631 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6035386 virtual) 2018-03-26 15:29:38,271 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6238850 virtual) 2018-03-26 15:29:38,743 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6383697 virtual) 2018-03-26 15:29:42,538 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7994845 virtual) 2018-03-26 15:29:43,507 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8440432 virtual) 2018-03-26 15:29:48,283 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8601355 virtual) 2018-03-26 15:29:48,613 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8834248 virtual) 2018-03-26 15:29:50,929 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8987398 virtual) 2018-03-26 15:29:51,716 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9127912 virtual) 2018-03-26 15:29:55,675 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9301229 virtual) 2018-03-26 15:30:01,120 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9459415 virtual) 2018-03-26 15:30:01,338 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9630777 virtual) 2018-03-26 15:30:05,044 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9809904 virtual) 2018-03-26 15:30:05,202 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9964183 virtual) 2018-03-26 15:30:09,043 : INFO : 43 batches submitted to accumulate stats from 2752 documents (10168878 virtual) 2018-03-26 15:30:09,422 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10344568 virtual) 2018-03-26 15:30:13,447 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10508976 virtual) 2018-03-26 15:30:13,712 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10654720 virtual) 2018-03-26 15:30:17,127 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10787315 virtual) 2018-03-26 15:30:18,617 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10982912 virtual) 2018-03-26 15:30:21,442 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11119440 virtual) 2018-03-26 15:30:22,566 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11346822 virtual) 2018-03-26 15:30:25,193 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11492957 virtual) 2018-03-26 15:30:25,947 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11625426 virtual) 2018-03-26 15:30:26,753 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11795143 virtual) 2018-03-26 15:30:29,293 : INFO : 54 batches submitted to accumulate stats from 3456 documents (11964309 virtual) 2018-03-26 15:30:30,036 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12093789 virtual) 2018-03-26 15:30:32,119 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12278591 virtual) 2018-03-26 15:30:32,882 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12441867 virtual) 2018-03-26 15:30:33,291 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12621963 virtual) 2018-03-26 15:30:36,091 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12759331 virtual) 2018-03-26 15:30:36,553 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12922397 virtual) 2018-03-26 15:30:37,204 : INFO : 61 batches submitted to accumulate stats from 3904 documents (13155554 virtual) 2018-03-26 15:30:40,564 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13291262 virtual) 2018-03-26 15:30:40,786 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13417251 virtual) 2018-03-26 15:30:41,545 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13568703 virtual) 2018-03-26 15:30:43,846 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13703987 virtual) 2018-03-26 15:30:44,725 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13834172 virtual) 2018-03-26 15:30:47,102 : INFO : 67 batches submitted to accumulate stats from 4288 documents (13978931 virtual) 2018-03-26 15:30:47,304 : INFO : 68 batches submitted to accumulate stats from 4352 documents (14179666 virtual) 2018-03-26 15:30:47,853 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14345605 virtual) 2018-03-26 15:30:50,714 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14499576 virtual) 2018-03-26 15:30:50,840 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14708829 virtual) 2018-03-26 15:30:51,159 : INFO : 72 batches submitted to accumulate stats from 4608 documents (14856037 virtual) 2018-03-26 15:30:54,375 : INFO : 73 batches submitted to accumulate stats from 4672 documents (15080012 virtual) 2018-03-26 15:30:55,265 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15264703 virtual) 2018-03-26 15:30:55,756 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15424104 virtual) 2018-03-26 15:30:58,276 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15586947 virtual) 2018-03-26 15:30:59,103 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15742355 virtual) 2018-03-26 15:31:00,263 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15895904 virtual) 2018-03-26 15:31:03,392 : INFO : 79 batches submitted to accumulate stats from 5056 documents (16067361 virtual) 2018-03-26 15:31:03,735 : INFO : 80 batches submitted to accumulate stats from 5120 documents (16246931 virtual) 2018-03-26 15:31:04,302 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16383751 virtual) 2018-03-26 15:31:07,496 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16530128 virtual) 2018-03-26 15:31:07,627 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16714976 virtual) 2018-03-26 15:31:08,154 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16899502 virtual) 2018-03-26 15:31:11,566 : INFO : 85 batches submitted to accumulate stats from 5440 documents (17054656 virtual) 2018-03-26 15:31:11,799 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17309811 virtual) 2018-03-26 15:31:12,155 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17476654 virtual) 2018-03-26 15:31:15,263 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17736596 virtual) 2018-03-26 15:31:16,373 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17874860 virtual) 2018-03-26 15:31:16,739 : INFO : 90 batches submitted to accumulate stats from 5760 documents (18022471 virtual) 2018-03-26 15:31:19,000 : INFO : 91 batches submitted to accumulate stats from 5824 documents (18180010 virtual) 2018-03-26 15:31:20,752 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18357946 virtual) 2018-03-26 15:31:22,106 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18533114 virtual) 2018-03-26 15:31:24,120 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18812149 virtual) 2018-03-26 15:31:25,452 : INFO : 95 batches submitted to accumulate stats from 6080 documents (18965818 virtual) 2018-03-26 15:31:25,837 : INFO : 96 batches submitted to accumulate stats from 6144 documents (19097137 virtual) 2018-03-26 15:31:28,062 : INFO : 97 batches submitted to accumulate stats from 6208 documents (19260684 virtual) 2018-03-26 15:31:29,778 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19409522 virtual) 2018-03-26 15:31:30,175 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19538574 virtual) 2018-03-26 15:31:33,391 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19733327 virtual) 2018-03-26 15:31:33,727 : INFO : 101 batches submitted to accumulate stats from 6464 documents (19928441 virtual) 2018-03-26 15:31:34,678 : INFO : 102 batches submitted to accumulate stats from 6528 documents (20152652 virtual) 2018-03-26 15:31:37,404 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20355016 virtual) 2018-03-26 15:31:37,511 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20521806 virtual) 2018-03-26 15:31:37,985 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20680233 virtual) 2018-03-26 15:31:42,184 : INFO : 106 batches submitted to accumulate stats from 6784 documents (20818595 virtual) 2018-03-26 15:31:43,106 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22415151 virtual) 2018-03-26 15:31:43,716 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22525052 virtual) 2018-03-26 15:31:46,638 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22709560 virtual) 2018-03-26 15:31:47,408 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22879966 virtual) 2018-03-26 15:31:47,617 : INFO : 111 batches submitted to accumulate stats from 7104 documents (23019450 virtual) 2018-03-26 15:31:50,111 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23219545 virtual) 2018-03-26 15:31:50,416 : INFO : 113 batches submitted to accumulate stats from 7232 documents (23520532 virtual) 2018-03-26 15:31:54,571 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23700860 virtual) 2018-03-26 15:31:54,824 : INFO : 115 batches submitted to accumulate stats from 7360 documents (23972854 virtual) 2018-03-26 15:31:58,133 : INFO : 116 batches submitted to accumulate stats from 7424 documents (24125455 virtual) 2018-03-26 15:31:58,908 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24322261 virtual) 2018-03-26 15:32:03,432 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24507532 virtual) 2018-03-26 15:32:05,437 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24680300 virtual) 2018-03-26 15:32:09,273 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24893668 virtual) 2018-03-26 15:32:09,862 : INFO : 121 batches submitted to accumulate stats from 7744 documents (24995758 virtual) 2018-03-26 15:32:14,176 : INFO : 122 batches submitted to accumulate stats from 7808 documents (25118479 virtual) 2018-03-26 15:32:14,456 : INFO : 123 batches submitted to accumulate stats from 7872 documents (25296625 virtual) 2018-03-26 15:32:18,489 : INFO : 124 batches submitted to accumulate stats from 7936 documents (25449426 virtual) 2018-03-26 15:32:19,576 : INFO : 125 batches submitted to accumulate stats from 8000 documents (25551342 virtual) 2018-03-26 15:32:25,435 : INFO : serializing accumulator to return to master... 2018-03-26 15:32:25,481 : INFO : accumulator serialized 2018-03-26 15:32:26,102 : INFO : serializing accumulator to return to master... 2018-03-26 15:32:26,107 : INFO : accumulator serialized 2018-03-26 15:32:27,023 : INFO : serializing accumulator to return to master... 2018-03-26 15:32:27,029 : INFO : accumulator serialized 2018-03-26 15:32:27,337 : INFO : 3 accumulators retrieved from output queue 2018-03-26 15:32:27,652 : INFO : accumulated word occurrence stats for 25551718 virtual documents 2018-03-26 15:32:28,677 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_40 2018-03-26 15:32:28,739 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-26 15:32:28,919 : INFO : setting ignored attribute state to None 2018-03-26 15:32:28,920 : INFO : setting ignored attribute dispatcher to None 2018-03-26 15:32:28,920 : INFO : setting ignored attribute id2word to None 2018-03-26 15:32:28,921 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_40 2018-03-26 15:32:28,922 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_40.state 2018-03-26 15:32:29,260 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_40.state 2018-03-26 15:32:29,336 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 15:32:29,454 : INFO : 1 batches submitted to accumulate stats from 64 documents (197185 virtual) 2018-03-26 15:32:29,513 : INFO : 2 batches submitted to accumulate stats from 128 documents (362915 virtual) 2018-03-26 15:32:29,583 : INFO : 3 batches submitted to accumulate stats from 192 documents (517020 virtual) 2018-03-26 15:32:29,777 : INFO : 4 batches submitted to accumulate stats from 256 documents (818899 virtual) 2018-03-26 15:32:29,882 : INFO : 5 batches submitted to accumulate stats from 320 documents (992042 virtual) 2018-03-26 15:32:29,981 : INFO : 6 batches submitted to accumulate stats from 384 documents (1153098 virtual) 2018-03-26 15:32:33,596 : INFO : 7 batches submitted to accumulate stats from 448 documents (1307015 virtual) 2018-03-26 15:32:33,737 : INFO : 8 batches submitted to accumulate stats from 512 documents (1495827 virtual) 2018-03-26 15:32:34,665 : INFO : 9 batches submitted to accumulate stats from 576 documents (1680936 virtual) 2018-03-26 15:32:38,158 : INFO : 10 batches submitted to accumulate stats from 640 documents (1818795 virtual) 2018-03-26 15:32:38,843 : INFO : 11 batches submitted to accumulate stats from 704 documents (1997044 virtual) 2018-03-26 15:32:41,737 : INFO : 12 batches submitted to accumulate stats from 768 documents (2167325 virtual) 2018-03-26 15:32:42,326 : INFO : 13 batches submitted to accumulate stats from 832 documents (2336675 virtual) 2018-03-26 15:32:44,342 : INFO : 14 batches submitted to accumulate stats from 896 documents (2455038 virtual) 2018-03-26 15:32:46,177 : INFO : 15 batches submitted to accumulate stats from 960 documents (2597171 virtual) 2018-03-26 15:32:46,798 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2966138 virtual) 2018-03-26 15:32:49,359 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3154744 virtual) 2018-03-26 15:32:50,510 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3338929 virtual) 2018-03-26 15:32:50,969 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3568693 virtual) 2018-03-26 15:32:52,406 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3750983 virtual) 2018-03-26 15:32:54,259 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3924850 virtual) 2018-03-26 15:32:57,336 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4108284 virtual) 2018-03-26 15:32:58,765 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4933350 virtual) 2018-03-26 15:33:00,135 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5083664 virtual) 2018-03-26 15:33:03,223 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5305423 virtual) 2018-03-26 15:33:03,498 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5444364 virtual) 2018-03-26 15:33:04,421 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5648101 virtual) 2018-03-26 15:33:07,741 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5800334 virtual) 2018-03-26 15:33:08,114 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6035465 virtual) 2018-03-26 15:33:11,574 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6241632 virtual) 2018-03-26 15:33:13,113 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6387021 virtual) 2018-03-26 15:33:16,898 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7994908 virtual) 2018-03-26 15:33:17,180 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8440443 virtual) 2018-03-26 15:33:22,192 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8603552 virtual) 2018-03-26 15:33:22,743 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8834324 virtual) 2018-03-26 15:33:23,916 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8995531 virtual) 2018-03-26 15:33:25,821 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9128128 virtual) 2018-03-26 15:33:29,951 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9316937 virtual) 2018-03-26 15:33:34,493 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9459508 virtual) 2018-03-26 15:33:35,612 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9631852 virtual) 2018-03-26 15:33:38,729 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9812272 virtual) 2018-03-26 15:33:38,818 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9966838 virtual) 2018-03-26 15:33:42,396 : INFO : 43 batches submitted to accumulate stats from 2752 documents (10173420 virtual) 2018-03-26 15:33:43,421 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10360996 virtual) 2018-03-26 15:33:46,770 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10509180 virtual) 2018-03-26 15:33:48,018 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10654935 virtual) 2018-03-26 15:33:50,678 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10795081 virtual) 2018-03-26 15:33:53,303 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10983050 virtual) 2018-03-26 15:33:55,329 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11125204 virtual) 2018-03-26 15:33:57,083 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11347041 virtual) 2018-03-26 15:33:59,086 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11495983 virtual) 2018-03-26 15:34:00,212 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11625670 virtual) 2018-03-26 15:34:00,741 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11795182 virtual) 2018-03-26 15:34:03,758 : INFO : 54 batches submitted to accumulate stats from 3456 documents (11970074 virtual) 2018-03-26 15:34:03,888 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12101552 virtual) 2018-03-26 15:34:06,259 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12282919 virtual) 2018-03-26 15:34:07,189 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12443707 virtual) 2018-03-26 15:34:07,549 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12623005 virtual) 2018-03-26 15:34:10,507 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12762968 virtual) 2018-03-26 15:34:10,887 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12928833 virtual) 2018-03-26 15:34:11,697 : INFO : 61 batches submitted to accumulate stats from 3904 documents (13156504 virtual) 2018-03-26 15:34:14,915 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13292187 virtual) 2018-03-26 15:34:15,093 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13420339 virtual) 2018-03-26 15:34:16,321 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13571269 virtual) 2018-03-26 15:34:18,489 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13710733 virtual) 2018-03-26 15:34:19,185 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13855020 virtual) 2018-03-26 15:34:21,996 : INFO : 67 batches submitted to accumulate stats from 4288 documents (13986368 virtual) 2018-03-26 15:34:22,116 : INFO : 68 batches submitted to accumulate stats from 4352 documents (14191309 virtual) 2018-03-26 15:34:22,407 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14345969 virtual) 2018-03-26 15:34:25,597 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14499803 virtual) 2018-03-26 15:34:25,780 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14710725 virtual) 2018-03-26 15:34:26,278 : INFO : 72 batches submitted to accumulate stats from 4608 documents (14860052 virtual) 2018-03-26 15:34:28,952 : INFO : 73 batches submitted to accumulate stats from 4672 documents (15083751 virtual) 2018-03-26 15:34:30,218 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15267838 virtual) 2018-03-26 15:34:30,871 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15435513 virtual) 2018-03-26 15:34:33,011 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15603628 virtual) 2018-03-26 15:34:34,549 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15752131 virtual) 2018-03-26 15:34:35,351 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15902912 virtual) 2018-03-26 15:34:38,581 : INFO : 79 batches submitted to accumulate stats from 5056 documents (16073394 virtual) 2018-03-26 15:34:39,024 : INFO : 80 batches submitted to accumulate stats from 5120 documents (16247327 virtual) 2018-03-26 15:34:39,749 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16387680 virtual) 2018-03-26 15:34:42,684 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16532816 virtual) 2018-03-26 15:34:43,053 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16721341 virtual) 2018-03-26 15:34:43,656 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16903970 virtual) 2018-03-26 15:34:47,082 : INFO : 85 batches submitted to accumulate stats from 5440 documents (17054752 virtual) 2018-03-26 15:34:47,221 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17313182 virtual) 2018-03-26 15:34:47,644 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17497031 virtual) 2018-03-26 15:34:50,854 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17756251 virtual) 2018-03-26 15:34:51,906 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17882281 virtual) 2018-03-26 15:34:52,388 : INFO : 90 batches submitted to accumulate stats from 5760 documents (18030235 virtual) 2018-03-26 15:34:54,584 : INFO : 91 batches submitted to accumulate stats from 5824 documents (18183334 virtual) 2018-03-26 15:34:56,863 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18358148 virtual) 2018-03-26 15:34:58,050 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18537192 virtual) 2018-03-26 15:35:00,101 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18825075 virtual) 2018-03-26 15:35:00,941 : INFO : 95 batches submitted to accumulate stats from 6080 documents (18974238 virtual) 2018-03-26 15:35:01,921 : INFO : 96 batches submitted to accumulate stats from 6144 documents (19108625 virtual) 2018-03-26 15:35:04,043 : INFO : 97 batches submitted to accumulate stats from 6208 documents (19271087 virtual) 2018-03-26 15:35:05,403 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19409681 virtual) 2018-03-26 15:35:06,445 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19538734 virtual) 2018-03-26 15:35:09,252 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19738500 virtual) 2018-03-26 15:35:09,813 : INFO : 101 batches submitted to accumulate stats from 6464 documents (19928566 virtual) 2018-03-26 15:35:11,362 : INFO : 102 batches submitted to accumulate stats from 6528 documents (20152773 virtual) 2018-03-26 15:35:13,216 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20355194 virtual) 2018-03-26 15:35:13,504 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20522007 virtual) 2018-03-26 15:35:14,772 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20680305 virtual) 2018-03-26 15:35:18,343 : INFO : 106 batches submitted to accumulate stats from 6784 documents (20818626 virtual) 2018-03-26 15:35:19,264 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22415254 virtual) 2018-03-26 15:35:20,539 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22525143 virtual) 2018-03-26 15:35:23,233 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22715405 virtual) 2018-03-26 15:35:24,101 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22880298 virtual) 2018-03-26 15:35:25,035 : INFO : 111 batches submitted to accumulate stats from 7104 documents (23019449 virtual) 2018-03-26 15:35:26,947 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23219678 virtual) 2018-03-26 15:35:28,048 : INFO : 113 batches submitted to accumulate stats from 7232 documents (23521598 virtual) 2018-03-26 15:35:31,754 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23701020 virtual) 2018-03-26 15:35:32,158 : INFO : 115 batches submitted to accumulate stats from 7360 documents (23972959 virtual) 2018-03-26 15:35:35,342 : INFO : 116 batches submitted to accumulate stats from 7424 documents (24126673 virtual) 2018-03-26 15:35:36,400 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24322339 virtual) 2018-03-26 15:35:40,691 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24517930 virtual) 2018-03-26 15:35:42,819 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24682945 virtual) 2018-03-26 15:35:46,630 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24893766 virtual) 2018-03-26 15:35:47,298 : INFO : 121 batches submitted to accumulate stats from 7744 documents (24996989 virtual) 2018-03-26 15:35:51,448 : INFO : 122 batches submitted to accumulate stats from 7808 documents (25118549 virtual) 2018-03-26 15:35:52,257 : INFO : 123 batches submitted to accumulate stats from 7872 documents (25296642 virtual) 2018-03-26 15:35:55,491 : INFO : 124 batches submitted to accumulate stats from 7936 documents (25449443 virtual) 2018-03-26 15:35:57,578 : INFO : 125 batches submitted to accumulate stats from 8000 documents (25551359 virtual) 2018-03-26 15:36:02,355 : INFO : serializing accumulator to return to master... 2018-03-26 15:36:02,442 : INFO : accumulator serialized 2018-03-26 15:36:03,963 : INFO : serializing accumulator to return to master... 2018-03-26 15:36:04,091 : INFO : serializing accumulator to return to master... 2018-03-26 15:36:03,970 : INFO : accumulator serialized 2018-03-26 15:36:04,097 : INFO : accumulator serialized 2018-03-26 15:36:04,467 : INFO : 3 accumulators retrieved from output queue 2018-03-26 15:36:04,850 : INFO : accumulated word occurrence stats for 25551734 virtual documents 2018-03-26 15:36:06,018 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_50 2018-03-26 15:36:06,089 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_50.expElogbeta.npy with mmap=None 2018-03-26 15:36:06,295 : INFO : setting ignored attribute state to None 2018-03-26 15:36:06,296 : INFO : setting ignored attribute dispatcher to None 2018-03-26 15:36:06,297 : INFO : setting ignored attribute id2word to None 2018-03-26 15:36:06,297 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_50 2018-03-26 15:36:06,298 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_50.state 2018-03-26 15:36:06,719 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_50.state 2018-03-26 15:36:06,812 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 15:36:06,930 : INFO : 1 batches submitted to accumulate stats from 64 documents (197185 virtual) 2018-03-26 15:36:06,990 : INFO : 2 batches submitted to accumulate stats from 128 documents (362915 virtual) 2018-03-26 15:36:07,060 : INFO : 3 batches submitted to accumulate stats from 192 documents (517020 virtual) 2018-03-26 15:36:07,253 : INFO : 4 batches submitted to accumulate stats from 256 documents (818899 virtual) 2018-03-26 15:36:07,363 : INFO : 5 batches submitted to accumulate stats from 320 documents (992042 virtual) 2018-03-26 15:36:07,458 : INFO : 6 batches submitted to accumulate stats from 384 documents (1148008 virtual) 2018-03-26 15:36:11,244 : INFO : 7 batches submitted to accumulate stats from 448 documents (1303579 virtual) 2018-03-26 15:36:11,347 : INFO : 8 batches submitted to accumulate stats from 512 documents (1495798 virtual) 2018-03-26 15:36:12,340 : INFO : 9 batches submitted to accumulate stats from 576 documents (1680839 virtual) 2018-03-26 15:36:16,168 : INFO : 10 batches submitted to accumulate stats from 640 documents (1818739 virtual) 2018-03-26 15:36:16,273 : INFO : 11 batches submitted to accumulate stats from 704 documents (1988219 virtual) 2018-03-26 15:36:19,106 : INFO : 12 batches submitted to accumulate stats from 768 documents (2159902 virtual) 2018-03-26 15:36:20,138 : INFO : 13 batches submitted to accumulate stats from 832 documents (2331768 virtual) 2018-03-26 15:36:21,143 : INFO : 14 batches submitted to accumulate stats from 896 documents (2452628 virtual) 2018-03-26 15:36:23,861 : INFO : 15 batches submitted to accumulate stats from 960 documents (2594649 virtual) 2018-03-26 15:36:24,116 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2961373 virtual) 2018-03-26 15:36:25,659 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3154729 virtual) 2018-03-26 15:36:28,309 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3338706 virtual) 2018-03-26 15:36:28,571 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3554439 virtual) 2018-03-26 15:36:28,743 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3746307 virtual) 2018-03-26 15:36:32,048 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3914036 virtual) 2018-03-26 15:36:33,660 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4108274 virtual) 2018-03-26 15:36:36,830 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4933273 virtual) 2018-03-26 15:36:38,196 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5083560 virtual) 2018-03-26 15:36:39,333 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5301515 virtual) 2018-03-26 15:36:41,703 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5444357 virtual) 2018-03-26 15:36:42,706 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5648045 virtual) 2018-03-26 15:36:44,288 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5794388 virtual) 2018-03-26 15:36:46,416 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6035397 virtual) 2018-03-26 15:36:49,584 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6238861 virtual) 2018-03-26 15:36:50,191 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6383708 virtual) 2018-03-26 15:36:54,186 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7994856 virtual) 2018-03-26 15:36:55,172 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8440317 virtual) 2018-03-26 15:37:00,193 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8601377 virtual) 2018-03-26 15:37:00,403 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8834270 virtual) 2018-03-26 15:37:02,583 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8987420 virtual) 2018-03-26 15:37:03,940 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9128042 virtual) 2018-03-26 15:37:08,022 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9315588 virtual) 2018-03-26 15:37:12,682 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9459520 virtual) 2018-03-26 15:37:13,990 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9631709 virtual) 2018-03-26 15:37:16,581 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9809922 virtual) 2018-03-26 15:37:17,536 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9964201 virtual) 2018-03-26 15:37:21,260 : INFO : 43 batches submitted to accumulate stats from 2752 documents (10168896 virtual) 2018-03-26 15:37:21,363 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10344586 virtual) 2018-03-26 15:37:25,976 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10508994 virtual) 2018-03-26 15:37:26,060 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10654738 virtual) 2018-03-26 15:37:30,016 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10787333 virtual) 2018-03-26 15:37:31,167 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10982930 virtual) 2018-03-26 15:37:34,668 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11119458 virtual) 2018-03-26 15:37:35,430 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11337078 virtual) 2018-03-26 15:37:38,498 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11492850 virtual) 2018-03-26 15:37:38,973 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11625402 virtual) 2018-03-26 15:37:39,227 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11792633 virtual) 2018-03-26 15:37:42,588 : INFO : 54 batches submitted to accumulate stats from 3456 documents (11964334 virtual) 2018-03-26 15:37:43,489 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12093775 virtual) 2018-03-26 15:37:44,632 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12270682 virtual) 2018-03-26 15:37:46,642 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12440606 virtual) 2018-03-26 15:37:46,936 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12619087 virtual) 2018-03-26 15:37:48,889 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12759358 virtual) 2018-03-26 15:37:50,348 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12922424 virtual) 2018-03-26 15:37:51,266 : INFO : 61 batches submitted to accumulate stats from 3904 documents (13155689 virtual) 2018-03-26 15:37:53,636 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13291467 virtual) 2018-03-26 15:37:54,839 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13417423 virtual) 2018-03-26 15:37:55,922 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13571286 virtual) 2018-03-26 15:37:57,238 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13710694 virtual) 2018-03-26 15:37:59,234 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13854984 virtual) 2018-03-26 15:38:00,803 : INFO : 67 batches submitted to accumulate stats from 4288 documents (13978952 virtual) 2018-03-26 15:38:02,075 : INFO : 68 batches submitted to accumulate stats from 4352 documents (14179687 virtual) 2018-03-26 15:38:02,712 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14345626 virtual) 2018-03-26 15:38:04,966 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14499597 virtual) 2018-03-26 15:38:05,881 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14708850 virtual) 2018-03-26 15:38:06,988 : INFO : 72 batches submitted to accumulate stats from 4608 documents (14856058 virtual) 2018-03-26 15:38:08,422 : INFO : 73 batches submitted to accumulate stats from 4672 documents (15080125 virtual) 2018-03-26 15:38:11,135 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15264728 virtual) 2018-03-26 15:38:11,284 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15424261 virtual) 2018-03-26 15:38:12,631 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15593849 virtual) 2018-03-26 15:38:14,979 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15746917 virtual) 2018-03-26 15:38:16,422 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15896167 virtual) 2018-03-26 15:38:18,255 : INFO : 79 batches submitted to accumulate stats from 5056 documents (16067468 virtual) 2018-03-26 15:38:19,637 : INFO : 80 batches submitted to accumulate stats from 5120 documents (16246998 virtual) 2018-03-26 15:38:20,699 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16387498 virtual) 2018-03-26 15:38:22,765 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16532736 virtual) 2018-03-26 15:38:23,667 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16715085 virtual) 2018-03-26 15:38:24,784 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16903926 virtual) 2018-03-26 15:38:27,343 : INFO : 85 batches submitted to accumulate stats from 5440 documents (17054713 virtual) 2018-03-26 15:38:28,394 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17309907 virtual) 2018-03-26 15:38:28,533 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17492583 virtual) 2018-03-26 15:38:31,173 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17741016 virtual) 2018-03-26 15:38:33,165 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17876354 virtual) 2018-03-26 15:38:33,483 : INFO : 90 batches submitted to accumulate stats from 5760 documents (18024534 virtual) 2018-03-26 15:38:35,097 : INFO : 91 batches submitted to accumulate stats from 5824 documents (18183217 virtual) 2018-03-26 15:38:38,173 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18358028 virtual) 2018-03-26 15:38:39,196 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18533288 virtual) 2018-03-26 15:38:41,447 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18823897 virtual) 2018-03-26 15:38:41,843 : INFO : 95 batches submitted to accumulate stats from 6080 documents (18969878 virtual) 2018-03-26 15:38:43,071 : INFO : 96 batches submitted to accumulate stats from 6144 documents (19104108 virtual) 2018-03-26 15:38:45,683 : INFO : 97 batches submitted to accumulate stats from 6208 documents (19260766 virtual) 2018-03-26 15:38:46,352 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19409615 virtual) 2018-03-26 15:38:47,657 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19538592 virtual) 2018-03-26 15:38:50,175 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19733345 virtual) 2018-03-26 15:38:51,199 : INFO : 101 batches submitted to accumulate stats from 6464 documents (19928459 virtual) 2018-03-26 15:38:53,158 : INFO : 102 batches submitted to accumulate stats from 6528 documents (20152670 virtual) 2018-03-26 15:38:54,344 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20355034 virtual) 2018-03-26 15:38:55,213 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20521824 virtual) 2018-03-26 15:38:56,647 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20680251 virtual) 2018-03-26 15:38:59,543 : INFO : 106 batches submitted to accumulate stats from 6784 documents (20818613 virtual) 2018-03-26 15:39:00,443 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22415169 virtual) 2018-03-26 15:39:02,619 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22525070 virtual) 2018-03-26 15:39:04,903 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22706421 virtual) 2018-03-26 15:39:05,009 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22879970 virtual) 2018-03-26 15:39:06,640 : INFO : 111 batches submitted to accumulate stats from 7104 documents (23019004 virtual) 2018-03-26 15:39:08,540 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23219317 virtual) 2018-03-26 15:39:09,481 : INFO : 113 batches submitted to accumulate stats from 7232 documents (23514833 virtual) 2018-03-26 15:39:13,219 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23700700 virtual) 2018-03-26 15:39:14,029 : INFO : 115 batches submitted to accumulate stats from 7360 documents (23957550 virtual) 2018-03-26 15:39:16,914 : INFO : 116 batches submitted to accumulate stats from 7424 documents (24113385 virtual) 2018-03-26 15:39:18,550 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24309477 virtual) 2018-03-26 15:39:23,086 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24507498 virtual) 2018-03-26 15:39:24,471 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24680257 virtual) 2018-03-26 15:39:28,522 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24890878 virtual) 2018-03-26 15:39:29,494 : INFO : 121 batches submitted to accumulate stats from 7744 documents (24995774 virtual) 2018-03-26 15:39:33,913 : INFO : 122 batches submitted to accumulate stats from 7808 documents (25118495 virtual) 2018-03-26 15:39:34,397 : INFO : 123 batches submitted to accumulate stats from 7872 documents (25293629 virtual) 2018-03-26 15:39:38,216 : INFO : 124 batches submitted to accumulate stats from 7936 documents (25435348 virtual) 2018-03-26 15:39:39,933 : INFO : 125 batches submitted to accumulate stats from 8000 documents (25551374 virtual) 2018-03-26 15:39:45,429 : INFO : serializing accumulator to return to master... 2018-03-26 15:39:45,513 : INFO : accumulator serialized 2018-03-26 15:39:46,457 : INFO : serializing accumulator to return to master... 2018-03-26 15:39:46,593 : INFO : serializing accumulator to return to master... 2018-03-26 15:39:46,462 : INFO : accumulator serialized 2018-03-26 15:39:46,599 : INFO : accumulator serialized 2018-03-26 15:39:47,068 : INFO : 3 accumulators retrieved from output queue 2018-03-26 15:39:47,557 : INFO : accumulated word occurrence stats for 25551753 virtual documents
In [25]:
tmfull_score['c_uci'] = tmfull_score.apply(lambda row: get_score(row.num_topic, dict_tmfull, texts=text_tmfull, corpus=corpus_tmfull,
coherence='c_uci', model_dir='../src/tmfull'),
axis=1)
2018-03-26 14:46:53,734 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_10 2018-03-26 14:46:53,739 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-26 14:46:53,742 : INFO : setting ignored attribute state to None 2018-03-26 14:46:53,743 : INFO : setting ignored attribute dispatcher to None 2018-03-26 14:46:53,744 : INFO : setting ignored attribute id2word to None 2018-03-26 14:46:53,744 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_10 2018-03-26 14:46:53,745 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_10.state 2018-03-26 14:46:53,781 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_10.state 2018-03-26 14:46:53,809 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 14:46:53,920 : INFO : 1 batches submitted to accumulate stats from 64 documents (205745 virtual) 2018-03-26 14:46:53,977 : INFO : 2 batches submitted to accumulate stats from 128 documents (382316 virtual) 2018-03-26 14:46:54,045 : INFO : 3 batches submitted to accumulate stats from 192 documents (531745 virtual) 2018-03-26 14:46:54,222 : INFO : 4 batches submitted to accumulate stats from 256 documents (840148 virtual) 2018-03-26 14:46:54,327 : INFO : 5 batches submitted to accumulate stats from 320 documents (1025019 virtual) 2018-03-26 14:46:54,390 : INFO : 6 batches submitted to accumulate stats from 384 documents (1153899 virtual) 2018-03-26 14:46:56,808 : INFO : 7 batches submitted to accumulate stats from 448 documents (1327715 virtual) 2018-03-26 14:46:57,323 : INFO : 8 batches submitted to accumulate stats from 512 documents (1524767 virtual) 2018-03-26 14:46:57,931 : INFO : 9 batches submitted to accumulate stats from 576 documents (1709293 virtual) 2018-03-26 14:47:00,229 : INFO : 10 batches submitted to accumulate stats from 640 documents (1844925 virtual) 2018-03-26 14:47:00,907 : INFO : 11 batches submitted to accumulate stats from 704 documents (2014482 virtual) 2018-03-26 14:47:02,553 : INFO : 12 batches submitted to accumulate stats from 768 documents (2213113 virtual) 2018-03-26 14:47:03,502 : INFO : 13 batches submitted to accumulate stats from 832 documents (2372103 virtual) 2018-03-26 14:47:04,590 : INFO : 14 batches submitted to accumulate stats from 896 documents (2476002 virtual) 2018-03-26 14:47:05,976 : INFO : 15 batches submitted to accumulate stats from 960 documents (2723818 virtual) 2018-03-26 14:47:06,187 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3008674 virtual) 2018-03-26 14:47:08,020 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3209891 virtual) 2018-03-26 14:47:08,998 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3399182 virtual) 2018-03-26 14:47:09,843 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3638271 virtual) 2018-03-26 14:47:10,129 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3830552 virtual) 2018-03-26 14:47:13,768 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3970032 virtual) 2018-03-26 14:47:13,905 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4163782 virtual) 2018-03-26 14:47:15,206 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4969321 virtual) 2018-03-26 14:47:17,478 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5111576 virtual) 2018-03-26 14:47:18,905 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5353532 virtual) 2018-03-26 14:47:19,067 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5503645 virtual) 2018-03-26 14:47:20,568 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5708912 virtual) 2018-03-26 14:47:23,174 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5867249 virtual) 2018-03-26 14:47:23,439 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6171445 virtual) 2018-03-26 14:47:26,318 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6316898 virtual) 2018-03-26 14:47:27,875 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6446678 virtual) 2018-03-26 14:47:30,417 : INFO : 32 batches submitted to accumulate stats from 2048 documents (8074268 virtual) 2018-03-26 14:47:31,149 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8529651 virtual) 2018-03-26 14:47:33,895 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8769862 virtual) 2018-03-26 14:47:34,471 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8917752 virtual) 2018-03-26 14:47:35,966 : INFO : 36 batches submitted to accumulate stats from 2304 documents (9065584 virtual) 2018-03-26 14:47:36,312 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9247231 virtual) 2018-03-26 14:47:40,847 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9395695 virtual) 2018-03-26 14:47:43,297 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9568729 virtual) 2018-03-26 14:47:43,596 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9766589 virtual) 2018-03-26 14:47:46,137 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9906640 virtual) 2018-03-26 14:47:46,893 : INFO : 42 batches submitted to accumulate stats from 2688 documents (10116354 virtual) 2018-03-26 14:47:48,939 : INFO : 43 batches submitted to accumulate stats from 2752 documents (10272905 virtual) 2018-03-26 14:47:49,989 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10461556 virtual) 2018-03-26 14:47:52,574 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10620144 virtual) 2018-03-26 14:47:52,717 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10751135 virtual) 2018-03-26 14:47:55,455 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10940485 virtual) 2018-03-26 14:47:56,471 : INFO : 48 batches submitted to accumulate stats from 3072 documents (11108959 virtual) 2018-03-26 14:47:59,007 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11310566 virtual) 2018-03-26 14:47:59,529 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11476827 virtual) 2018-03-26 14:48:01,586 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11613637 virtual) 2018-03-26 14:48:02,958 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11772221 virtual) 2018-03-26 14:48:04,728 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11963629 virtual) 2018-03-26 14:48:05,802 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12093173 virtual) 2018-03-26 14:48:06,674 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12282107 virtual) 2018-03-26 14:48:08,040 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12442912 virtual) 2018-03-26 14:48:08,335 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12624133 virtual) 2018-03-26 14:48:09,571 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12768910 virtual) 2018-03-26 14:48:10,808 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12944936 virtual) 2018-03-26 14:48:11,725 : INFO : 60 batches submitted to accumulate stats from 3840 documents (13172549 virtual) 2018-03-26 14:48:13,081 : INFO : 61 batches submitted to accumulate stats from 3904 documents (13311908 virtual) 2018-03-26 14:48:13,910 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13446629 virtual) 2018-03-26 14:48:15,172 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13595319 virtual) 2018-03-26 14:48:15,974 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13736889 virtual) 2018-03-26 14:48:17,186 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13886373 virtual) 2018-03-26 14:48:18,515 : INFO : 66 batches submitted to accumulate stats from 4224 documents (14057576 virtual) 2018-03-26 14:48:19,371 : INFO : 67 batches submitted to accumulate stats from 4288 documents (14234755 virtual) 2018-03-26 14:48:19,724 : INFO : 68 batches submitted to accumulate stats from 4352 documents (14405106 virtual) 2018-03-26 14:48:21,196 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14533770 virtual) 2018-03-26 14:48:22,117 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14785998 virtual) 2018-03-26 14:48:22,647 : INFO : 71 batches submitted to accumulate stats from 4544 documents (15008017 virtual) 2018-03-26 14:48:24,577 : INFO : 72 batches submitted to accumulate stats from 4608 documents (15165684 virtual) 2018-03-26 14:48:25,404 : INFO : 73 batches submitted to accumulate stats from 4672 documents (15339083 virtual) 2018-03-26 14:48:25,800 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15528101 virtual) 2018-03-26 14:48:27,042 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15688896 virtual) 2018-03-26 14:48:29,789 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15829510 virtual) 2018-03-26 14:48:30,067 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15986959 virtual) 2018-03-26 14:48:30,172 : INFO : 78 batches submitted to accumulate stats from 4992 documents (16198071 virtual) 2018-03-26 14:48:32,962 : INFO : 79 batches submitted to accumulate stats from 5056 documents (16322769 virtual) 2018-03-26 14:48:33,239 : INFO : 80 batches submitted to accumulate stats from 5120 documents (16497897 virtual) 2018-03-26 14:48:33,681 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16660878 virtual) 2018-03-26 14:48:35,712 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16869728 virtual) 2018-03-26 14:48:36,253 : INFO : 83 batches submitted to accumulate stats from 5312 documents (17002029 virtual) 2018-03-26 14:48:37,652 : INFO : 84 batches submitted to accumulate stats from 5376 documents (17294721 virtual) 2018-03-26 14:48:38,085 : INFO : 85 batches submitted to accumulate stats from 5440 documents (17458212 virtual) 2018-03-26 14:48:39,678 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17707135 virtual) 2018-03-26 14:48:40,809 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17864077 virtual) 2018-03-26 14:48:42,021 : INFO : 88 batches submitted to accumulate stats from 5632 documents (18005836 virtual) 2018-03-26 14:48:42,110 : INFO : 89 batches submitted to accumulate stats from 5696 documents (18172814 virtual) 2018-03-26 14:48:45,019 : INFO : 90 batches submitted to accumulate stats from 5760 documents (18355589 virtual) 2018-03-26 14:48:45,836 : INFO : 91 batches submitted to accumulate stats from 5824 documents (18531676 virtual) 2018-03-26 14:48:46,834 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18810862 virtual) 2018-03-26 14:48:48,075 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18964531 virtual) 2018-03-26 14:48:48,518 : INFO : 94 batches submitted to accumulate stats from 6016 documents (19102801 virtual) 2018-03-26 14:48:50,296 : INFO : 95 batches submitted to accumulate stats from 6080 documents (19274255 virtual) 2018-03-26 14:48:51,510 : INFO : 96 batches submitted to accumulate stats from 6144 documents (19408478 virtual) 2018-03-26 14:48:51,689 : INFO : 97 batches submitted to accumulate stats from 6208 documents (19538049 virtual) 2018-03-26 14:48:54,235 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19741342 virtual) 2018-03-26 14:48:54,476 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19939037 virtual) 2018-03-26 14:48:55,701 : INFO : 100 batches submitted to accumulate stats from 6400 documents (20177663 virtual) 2018-03-26 14:48:57,156 : INFO : 101 batches submitted to accumulate stats from 6464 documents (20378731 virtual) 2018-03-26 14:48:57,443 : INFO : 102 batches submitted to accumulate stats from 6528 documents (20523755 virtual) 2018-03-26 14:48:58,079 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20699276 virtual) 2018-03-26 14:49:01,099 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20841879 virtual) 2018-03-26 14:49:01,899 : INFO : 105 batches submitted to accumulate stats from 6720 documents (22429558 virtual) 2018-03-26 14:49:02,966 : INFO : 106 batches submitted to accumulate stats from 6784 documents (22585757 virtual) 2018-03-26 14:49:03,973 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22754049 virtual) 2018-03-26 14:49:05,233 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22905743 virtual) 2018-03-26 14:49:06,299 : INFO : 109 batches submitted to accumulate stats from 6976 documents (23060741 virtual) 2018-03-26 14:49:06,716 : INFO : 110 batches submitted to accumulate stats from 7040 documents (23266613 virtual) 2018-03-26 14:49:09,186 : INFO : 111 batches submitted to accumulate stats from 7104 documents (23589025 virtual) 2018-03-26 14:49:09,881 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23738308 virtual) 2018-03-26 14:49:11,984 : INFO : 113 batches submitted to accumulate stats from 7232 documents (24030149 virtual) 2018-03-26 14:49:12,642 : INFO : 114 batches submitted to accumulate stats from 7296 documents (24201786 virtual) 2018-03-26 14:49:15,489 : INFO : 115 batches submitted to accumulate stats from 7360 documents (24393107 virtual) 2018-03-26 14:49:18,398 : INFO : 116 batches submitted to accumulate stats from 7424 documents (24581823 virtual) 2018-03-26 14:49:18,557 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24761103 virtual) 2018-03-26 14:49:21,678 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24964643 virtual) 2018-03-26 14:49:23,666 : INFO : 119 batches submitted to accumulate stats from 7616 documents (25050729 virtual) 2018-03-26 14:49:25,084 : INFO : 120 batches submitted to accumulate stats from 7680 documents (25201726 virtual) 2018-03-26 14:49:27,366 : INFO : 121 batches submitted to accumulate stats from 7744 documents (25373821 virtual) 2018-03-26 14:49:28,491 : INFO : 122 batches submitted to accumulate stats from 7808 documents (25548267 virtual) 2018-03-26 14:49:30,245 : INFO : 123 batches submitted to accumulate stats from 7872 documents (25549770 virtual) 2018-03-26 14:49:34,331 : INFO : serializing accumulator to return to master... 2018-03-26 14:49:34,337 : INFO : accumulator serialized 2018-03-26 14:49:35,664 : INFO : serializing accumulator to return to master... 2018-03-26 14:49:35,669 : INFO : accumulator serialized 2018-03-26 14:49:35,759 : INFO : serializing accumulator to return to master... 2018-03-26 14:49:35,765 : INFO : accumulator serialized 2018-03-26 14:49:35,913 : INFO : 3 accumulators retrieved from output queue 2018-03-26 14:49:35,968 : INFO : accumulated word occurrence stats for 25549991 virtual documents 2018-03-26 14:49:36,194 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_15 2018-03-26 14:49:36,242 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_15.expElogbeta.npy with mmap=None 2018-03-26 14:49:36,312 : INFO : setting ignored attribute state to None 2018-03-26 14:49:36,313 : INFO : setting ignored attribute dispatcher to None 2018-03-26 14:49:36,313 : INFO : setting ignored attribute id2word to None 2018-03-26 14:49:36,314 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_15 2018-03-26 14:49:36,315 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_15.state 2018-03-26 14:49:36,456 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_15.state 2018-03-26 14:49:36,499 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 14:49:36,610 : INFO : 1 batches submitted to accumulate stats from 64 documents (197181 virtual) 2018-03-26 14:49:36,668 : INFO : 2 batches submitted to accumulate stats from 128 documents (371824 virtual) 2018-03-26 14:49:36,732 : INFO : 3 batches submitted to accumulate stats from 192 documents (520698 virtual) 2018-03-26 14:49:36,905 : INFO : 4 batches submitted to accumulate stats from 256 documents (831326 virtual) 2018-03-26 14:49:36,996 : INFO : 5 batches submitted to accumulate stats from 320 documents (993683 virtual) 2018-03-26 14:49:37,088 : INFO : 6 batches submitted to accumulate stats from 384 documents (1153316 virtual) 2018-03-26 14:49:39,928 : INFO : 7 batches submitted to accumulate stats from 448 documents (1320992 virtual) 2018-03-26 14:49:40,055 : INFO : 8 batches submitted to accumulate stats from 512 documents (1508960 virtual) 2018-03-26 14:49:41,017 : INFO : 9 batches submitted to accumulate stats from 576 documents (1695982 virtual) 2018-03-26 14:49:43,494 : INFO : 10 batches submitted to accumulate stats from 640 documents (1833108 virtual) 2018-03-26 14:49:44,344 : INFO : 11 batches submitted to accumulate stats from 704 documents (2008256 virtual) 2018-03-26 14:49:46,243 : INFO : 12 batches submitted to accumulate stats from 768 documents (2184809 virtual) 2018-03-26 14:49:46,865 : INFO : 13 batches submitted to accumulate stats from 832 documents (2366452 virtual) 2018-03-26 14:49:48,302 : INFO : 14 batches submitted to accumulate stats from 896 documents (2466651 virtual) 2018-03-26 14:49:49,761 : INFO : 15 batches submitted to accumulate stats from 960 documents (2653376 virtual) 2018-03-26 14:49:50,125 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2998011 virtual) 2018-03-26 14:49:51,996 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3187743 virtual) 2018-03-26 14:49:53,513 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3390841 virtual) 2018-03-26 14:49:53,841 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3609527 virtual) 2018-03-26 14:49:54,023 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3786504 virtual) 2018-03-26 14:49:57,678 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3952233 virtual) 2018-03-26 14:49:57,796 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4143812 virtual) 2018-03-26 14:50:01,649 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4957376 virtual) 2018-03-26 14:50:02,169 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5092919 virtual) 2018-03-26 14:50:02,690 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5330714 virtual) 2018-03-26 14:50:05,404 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5473366 virtual) 2018-03-26 14:50:05,830 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5680177 virtual) 2018-03-26 14:50:06,693 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5837768 virtual) 2018-03-26 14:50:08,524 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6083432 virtual) 2018-03-26 14:50:11,394 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6277636 virtual) 2018-03-26 14:50:11,598 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6416413 virtual) 2018-03-26 14:50:15,079 : INFO : 32 batches submitted to accumulate stats from 2048 documents (8023118 virtual) 2018-03-26 14:50:15,998 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8480544 virtual) 2018-03-26 14:50:19,839 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8719608 virtual) 2018-03-26 14:50:19,950 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8873322 virtual) 2018-03-26 14:50:22,194 : INFO : 36 batches submitted to accumulate stats from 2304 documents (9025766 virtual) 2018-03-26 14:50:22,533 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9176980 virtual) 2018-03-26 14:50:27,388 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9353748 virtual) 2018-03-26 14:50:30,446 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9523820 virtual) 2018-03-26 14:50:30,557 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9720846 virtual) 2018-03-26 14:50:33,592 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9869752 virtual) 2018-03-26 14:50:33,686 : INFO : 42 batches submitted to accumulate stats from 2688 documents (10041438 virtual) 2018-03-26 14:50:37,043 : INFO : 43 batches submitted to accumulate stats from 2752 documents (10220531 virtual) 2018-03-26 14:50:37,186 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10410056 virtual) 2018-03-26 14:50:40,238 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10578271 virtual) 2018-03-26 14:50:41,268 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10708563 virtual) 2018-03-26 14:50:43,760 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10874519 virtual) 2018-03-26 14:50:45,079 : INFO : 48 batches submitted to accumulate stats from 3072 documents (11039513 virtual) 2018-03-26 14:50:47,445 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11230763 virtual) 2018-03-26 14:50:48,612 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11425799 virtual) 2018-03-26 14:50:50,211 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11554062 virtual) 2018-03-26 14:50:51,969 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11671614 virtual) 2018-03-26 14:50:52,169 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11881915 virtual) 2018-03-26 14:50:53,501 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12027218 virtual) 2018-03-26 14:50:55,863 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12202313 virtual) 2018-03-26 14:50:56,188 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12364436 virtual) 2018-03-26 14:50:56,364 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12542893 virtual) 2018-03-26 14:50:58,204 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12720670 virtual) 2018-03-26 14:50:59,489 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12859552 virtual) 2018-03-26 14:51:00,530 : INFO : 60 batches submitted to accumulate stats from 3840 documents (13055166 virtual) 2018-03-26 14:51:01,857 : INFO : 61 batches submitted to accumulate stats from 3904 documents (13231658 virtual) 2018-03-26 14:51:02,883 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13363326 virtual) 2018-03-26 14:51:04,195 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13532971 virtual) 2018-03-26 14:51:05,503 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13661154 virtual) 2018-03-26 14:51:05,727 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13801528 virtual) 2018-03-26 14:51:08,019 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13936423 virtual) 2018-03-26 14:51:08,517 : INFO : 67 batches submitted to accumulate stats from 4288 documents (14129736 virtual) 2018-03-26 14:51:09,202 : INFO : 68 batches submitted to accumulate stats from 4352 documents (14311306 virtual) 2018-03-26 14:51:11,161 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14479396 virtual) 2018-03-26 14:51:11,477 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14680738 virtual) 2018-03-26 14:51:12,266 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14829475 virtual) 2018-03-26 14:51:14,169 : INFO : 72 batches submitted to accumulate stats from 4608 documents (15064423 virtual) 2018-03-26 14:51:15,346 : INFO : 73 batches submitted to accumulate stats from 4672 documents (15230712 virtual) 2018-03-26 14:51:15,854 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15398969 virtual) 2018-03-26 14:51:17,620 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15579091 virtual) 2018-03-26 14:51:18,883 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15733186 virtual) 2018-03-26 14:51:19,490 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15885165 virtual) 2018-03-26 14:51:22,174 : INFO : 78 batches submitted to accumulate stats from 4992 documents (16050292 virtual) 2018-03-26 14:51:22,387 : INFO : 79 batches submitted to accumulate stats from 5056 documents (16235914 virtual) 2018-03-26 14:51:22,952 : INFO : 80 batches submitted to accumulate stats from 5120 documents (16369682 virtual) 2018-03-26 14:51:25,656 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16532356 virtual) 2018-03-26 14:51:25,943 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16714705 virtual) 2018-03-26 14:51:26,274 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16903546 virtual) 2018-03-26 14:51:29,051 : INFO : 84 batches submitted to accumulate stats from 5376 documents (17054394 virtual) 2018-03-26 14:51:29,194 : INFO : 85 batches submitted to accumulate stats from 5440 documents (17315733 virtual) 2018-03-26 14:51:29,819 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17567381 virtual) 2018-03-26 14:51:32,343 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17757116 virtual) 2018-03-26 14:51:33,042 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17898251 virtual) 2018-03-26 14:51:33,698 : INFO : 89 batches submitted to accumulate stats from 5696 documents (18035526 virtual) 2018-03-26 14:51:35,412 : INFO : 90 batches submitted to accumulate stats from 5760 documents (18183357 virtual) 2018-03-26 14:51:37,952 : INFO : 91 batches submitted to accumulate stats from 5824 documents (18384081 virtual) 2018-03-26 14:51:38,776 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18559585 virtual) 2018-03-26 14:51:39,477 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18855926 virtual) 2018-03-26 14:51:41,017 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18992268 virtual) 2018-03-26 14:51:41,647 : INFO : 95 batches submitted to accumulate stats from 6080 documents (19128024 virtual) 2018-03-26 14:51:42,504 : INFO : 96 batches submitted to accumulate stats from 6144 documents (19301225 virtual) 2018-03-26 14:51:45,151 : INFO : 97 batches submitted to accumulate stats from 6208 documents (19434891 virtual) 2018-03-26 14:51:45,356 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19557220 virtual) 2018-03-26 14:51:47,922 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19792961 virtual) 2018-03-26 14:51:48,132 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19977468 virtual) 2018-03-26 14:51:48,696 : INFO : 101 batches submitted to accumulate stats from 6464 documents (20186353 virtual) 2018-03-26 14:51:50,975 : INFO : 102 batches submitted to accumulate stats from 6528 documents (20401203 virtual) 2018-03-26 14:51:51,241 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20555148 virtual) 2018-03-26 14:51:51,596 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20716487 virtual) 2018-03-26 14:51:55,028 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20852427 virtual) 2018-03-26 14:51:55,985 : INFO : 106 batches submitted to accumulate stats from 6784 documents (22436848 virtual) 2018-03-26 14:51:56,194 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22587138 virtual) 2018-03-26 14:51:59,338 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22764968 virtual) 2018-03-26 14:51:59,487 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22910771 virtual) 2018-03-26 14:51:59,635 : INFO : 110 batches submitted to accumulate stats from 7040 documents (23061822 virtual) 2018-03-26 14:52:02,144 : INFO : 111 batches submitted to accumulate stats from 7104 documents (23267808 virtual) 2018-03-26 14:52:02,720 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23600451 virtual) 2018-03-26 14:52:05,630 : INFO : 113 batches submitted to accumulate stats from 7232 documents (23739337 virtual) 2018-03-26 14:52:05,953 : INFO : 114 batches submitted to accumulate stats from 7296 documents (24031088 virtual) 2018-03-26 14:52:08,667 : INFO : 115 batches submitted to accumulate stats from 7360 documents (24198969 virtual) 2018-03-26 14:52:09,631 : INFO : 116 batches submitted to accumulate stats from 7424 documents (24383937 virtual) 2018-03-26 14:52:12,447 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24580667 virtual) 2018-03-26 14:52:15,276 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24745891 virtual) 2018-03-26 14:52:18,130 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24948411 virtual) 2018-03-26 14:52:18,718 : INFO : 120 batches submitted to accumulate stats from 7680 documents (25047798 virtual) 2018-03-26 14:52:21,969 : INFO : 121 batches submitted to accumulate stats from 7744 documents (25193344 virtual) 2018-03-26 14:52:22,841 : INFO : 122 batches submitted to accumulate stats from 7808 documents (25367434 virtual) 2018-03-26 14:52:25,336 : INFO : 123 batches submitted to accumulate stats from 7872 documents (25525632 virtual) 2018-03-26 14:52:26,887 : INFO : 124 batches submitted to accumulate stats from 7936 documents (25550875 virtual) 2018-03-26 14:52:31,421 : INFO : serializing accumulator to return to master... 2018-03-26 14:52:31,426 : INFO : accumulator serialized 2018-03-26 14:52:31,842 : INFO : serializing accumulator to return to master... 2018-03-26 14:52:31,848 : INFO : accumulator serialized 2018-03-26 14:52:32,555 : INFO : serializing accumulator to return to master... 2018-03-26 14:52:32,562 : INFO : accumulator serialized 2018-03-26 14:52:32,721 : INFO : 3 accumulators retrieved from output queue 2018-03-26 14:52:32,799 : INFO : accumulated word occurrence stats for 25551129 virtual documents 2018-03-26 14:52:33,045 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_20 2018-03-26 14:52:33,332 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_20.expElogbeta.npy with mmap=None 2018-03-26 14:52:33,416 : INFO : setting ignored attribute state to None 2018-03-26 14:52:33,416 : INFO : setting ignored attribute dispatcher to None 2018-03-26 14:52:33,417 : INFO : setting ignored attribute id2word to None 2018-03-26 14:52:33,417 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_20 2018-03-26 14:52:33,418 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_20.state 2018-03-26 14:52:33,594 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_20.state 2018-03-26 14:52:33,650 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 14:52:33,761 : INFO : 1 batches submitted to accumulate stats from 64 documents (197185 virtual) 2018-03-26 14:52:33,816 : INFO : 2 batches submitted to accumulate stats from 128 documents (362915 virtual) 2018-03-26 14:52:33,888 : INFO : 3 batches submitted to accumulate stats from 192 documents (517020 virtual) 2018-03-26 14:52:34,076 : INFO : 4 batches submitted to accumulate stats from 256 documents (818899 virtual) 2018-03-26 14:52:34,181 : INFO : 5 batches submitted to accumulate stats from 320 documents (992042 virtual) 2018-03-26 14:52:34,279 : INFO : 6 batches submitted to accumulate stats from 384 documents (1153098 virtual) 2018-03-26 14:52:37,523 : INFO : 7 batches submitted to accumulate stats from 448 documents (1307015 virtual) 2018-03-26 14:52:37,625 : INFO : 8 batches submitted to accumulate stats from 512 documents (1495827 virtual) 2018-03-26 14:52:38,251 : INFO : 9 batches submitted to accumulate stats from 576 documents (1680936 virtual) 2018-03-26 14:52:41,568 : INFO : 10 batches submitted to accumulate stats from 640 documents (1818795 virtual) 2018-03-26 14:52:41,911 : INFO : 11 batches submitted to accumulate stats from 704 documents (2000866 virtual) 2018-03-26 14:52:44,523 : INFO : 12 batches submitted to accumulate stats from 768 documents (2171486 virtual) 2018-03-26 14:52:45,000 : INFO : 13 batches submitted to accumulate stats from 832 documents (2337797 virtual) 2018-03-26 14:52:46,264 : INFO : 14 batches submitted to accumulate stats from 896 documents (2457918 virtual) 2018-03-26 14:52:48,108 : INFO : 15 batches submitted to accumulate stats from 960 documents (2599989 virtual) 2018-03-26 14:52:48,697 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2972836 virtual) 2018-03-26 14:52:50,337 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3154858 virtual) 2018-03-26 14:52:51,830 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3339062 virtual) 2018-03-26 14:52:52,318 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3569027 virtual) 2018-03-26 14:52:53,022 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3755563 virtual) 2018-03-26 14:52:55,130 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3925090 virtual) 2018-03-26 14:52:57,087 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4109915 virtual) 2018-03-26 14:52:59,192 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4938588 virtual) 2018-03-26 14:53:00,576 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5084129 virtual) 2018-03-26 14:53:02,238 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5307230 virtual) 2018-03-26 14:53:03,220 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5447532 virtual) 2018-03-26 14:53:04,059 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5659758 virtual) 2018-03-26 14:53:06,387 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5815410 virtual) 2018-03-26 14:53:07,029 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6048242 virtual) 2018-03-26 14:53:10,126 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6252659 virtual) 2018-03-26 14:53:11,130 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6403564 virtual) 2018-03-26 14:53:14,564 : INFO : 32 batches submitted to accumulate stats from 2048 documents (8006647 virtual) 2018-03-26 14:53:15,177 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8443004 virtual) 2018-03-26 14:53:19,422 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8621759 virtual) 2018-03-26 14:53:19,669 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8859278 virtual) 2018-03-26 14:53:21,226 : INFO : 36 batches submitted to accumulate stats from 2304 documents (9012192 virtual) 2018-03-26 14:53:22,709 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9145901 virtual) 2018-03-26 14:53:26,702 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9337382 virtual) 2018-03-26 14:53:29,824 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9488807 virtual) 2018-03-26 14:53:31,821 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9657314 virtual) 2018-03-26 14:53:33,138 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9828004 virtual) 2018-03-26 14:53:34,619 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9990733 virtual) 2018-03-26 14:53:37,314 : INFO : 43 batches submitted to accumulate stats from 2752 documents (10207744 virtual) 2018-03-26 14:53:37,987 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10384097 virtual) 2018-03-26 14:53:40,990 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10548164 virtual) 2018-03-26 14:53:41,961 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10697370 virtual) 2018-03-26 14:53:44,615 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10844108 virtual) 2018-03-26 14:53:46,511 : INFO : 48 batches submitted to accumulate stats from 3072 documents (11005488 virtual) 2018-03-26 14:53:48,389 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11166308 virtual) 2018-03-26 14:53:50,142 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11382859 virtual) 2018-03-26 14:53:51,744 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11508417 virtual) 2018-03-26 14:53:53,162 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11640444 virtual) 2018-03-26 14:53:53,459 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11843725 virtual) 2018-03-26 14:53:55,153 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12001289 virtual) 2018-03-26 14:53:56,626 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12161422 virtual) 2018-03-26 14:53:57,909 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12336949 virtual) 2018-03-26 14:53:58,213 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12499054 virtual) 2018-03-26 14:53:59,706 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12680189 virtual) 2018-03-26 14:54:01,616 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12807576 virtual) 2018-03-26 14:54:02,427 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12997357 virtual) 2018-03-26 14:54:03,240 : INFO : 61 batches submitted to accumulate stats from 3904 documents (13198413 virtual) 2018-03-26 14:54:05,448 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13331875 virtual) 2018-03-26 14:54:06,088 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13481361 virtual) 2018-03-26 14:54:07,197 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13617558 virtual) 2018-03-26 14:54:08,321 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13751490 virtual) 2018-03-26 14:54:10,204 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13901323 virtual) 2018-03-26 14:54:11,191 : INFO : 67 batches submitted to accumulate stats from 4288 documents (14073296 virtual) 2018-03-26 14:54:11,752 : INFO : 68 batches submitted to accumulate stats from 4352 documents (14237466 virtual) 2018-03-26 14:54:13,470 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14420388 virtual) 2018-03-26 14:54:14,230 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14537142 virtual) 2018-03-26 14:54:14,671 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14790947 virtual) 2018-03-26 14:54:16,772 : INFO : 72 batches submitted to accumulate stats from 4608 documents (15009178 virtual) 2018-03-26 14:54:18,178 : INFO : 73 batches submitted to accumulate stats from 4672 documents (15166839 virtual) 2018-03-26 14:54:18,274 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15340159 virtual) 2018-03-26 14:54:20,689 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15529173 virtual) 2018-03-26 14:54:20,870 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15682942 virtual) 2018-03-26 14:54:23,592 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15816972 virtual) 2018-03-26 14:54:24,133 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15967399 virtual) 2018-03-26 14:54:25,609 : INFO : 79 batches submitted to accumulate stats from 5056 documents (16181905 virtual) 2018-03-26 14:54:27,558 : INFO : 80 batches submitted to accumulate stats from 5120 documents (16299008 virtual) 2018-03-26 14:54:28,239 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16457542 virtual) 2018-03-26 14:54:28,908 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16622026 virtual) 2018-03-26 14:54:30,771 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16832894 virtual) 2018-03-26 14:54:31,617 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16966756 virtual) 2018-03-26 14:54:33,277 : INFO : 85 batches submitted to accumulate stats from 5440 documents (17255197 virtual) 2018-03-26 14:54:33,524 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17428635 virtual) 2018-03-26 14:54:35,285 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17680292 virtual) 2018-03-26 14:54:36,839 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17825909 virtual) 2018-03-26 14:54:38,235 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17955148 virtual) 2018-03-26 14:54:38,318 : INFO : 90 batches submitted to accumulate stats from 5760 documents (18112255 virtual) 2018-03-26 14:54:41,956 : INFO : 91 batches submitted to accumulate stats from 5824 documents (18305177 virtual) 2018-03-26 14:54:42,433 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18464232 virtual) 2018-03-26 14:54:43,754 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18640293 virtual) 2018-03-26 14:54:45,242 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18912293 virtual) 2018-03-26 14:54:45,386 : INFO : 95 batches submitted to accumulate stats from 6080 documents (19043497 virtual) 2018-03-26 14:54:47,303 : INFO : 96 batches submitted to accumulate stats from 6144 documents (19196459 virtual) 2018-03-26 14:54:48,889 : INFO : 97 batches submitted to accumulate stats from 6208 documents (19378518 virtual) 2018-03-26 14:54:49,460 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19496403 virtual) 2018-03-26 14:54:51,176 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19655295 virtual) 2018-03-26 14:54:52,430 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19857642 virtual) 2018-03-26 14:54:54,545 : INFO : 101 batches submitted to accumulate stats from 6464 documents (20072684 virtual) 2018-03-26 14:54:54,982 : INFO : 102 batches submitted to accumulate stats from 6528 documents (20298313 virtual) 2018-03-26 14:54:56,454 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20468689 virtual) 2018-03-26 14:54:57,217 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20651029 virtual) 2018-03-26 14:54:58,489 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20795144 virtual) 2018-03-26 14:55:01,162 : INFO : 106 batches submitted to accumulate stats from 6784 documents (22363377 virtual) 2018-03-26 14:55:02,156 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22485996 virtual) 2018-03-26 14:55:03,559 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22657475 virtual) 2018-03-26 14:55:05,094 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22827094 virtual) 2018-03-26 14:55:06,104 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22980624 virtual) 2018-03-26 14:55:06,754 : INFO : 111 batches submitted to accumulate stats from 7104 documents (23189774 virtual) 2018-03-26 14:55:08,876 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23458525 virtual) 2018-03-26 14:55:10,534 : INFO : 113 batches submitted to accumulate stats from 7232 documents (23678204 virtual) 2018-03-26 14:55:12,556 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23926483 virtual) 2018-03-26 14:55:13,850 : INFO : 115 batches submitted to accumulate stats from 7360 documents (24087817 virtual) 2018-03-26 14:55:16,497 : INFO : 116 batches submitted to accumulate stats from 7424 documents (24274628 virtual) 2018-03-26 14:55:19,451 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24443802 virtual) 2018-03-26 14:55:21,776 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24643750 virtual) 2018-03-26 14:55:25,115 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24833799 virtual) 2018-03-26 14:55:25,565 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24988366 virtual) 2018-03-26 14:55:29,088 : INFO : 121 batches submitted to accumulate stats from 7744 documents (25077991 virtual) 2018-03-26 14:55:29,345 : INFO : 122 batches submitted to accumulate stats from 7808 documents (25248223 virtual) 2018-03-26 14:55:33,375 : INFO : 123 batches submitted to accumulate stats from 7872 documents (25421880 virtual) 2018-03-26 14:55:33,465 : INFO : 124 batches submitted to accumulate stats from 7936 documents (25551237 virtual) 2018-03-26 14:55:39,044 : INFO : serializing accumulator to return to master... 2018-03-26 14:55:39,050 : INFO : accumulator serialized 2018-03-26 14:55:40,122 : INFO : serializing accumulator to return to master... 2018-03-26 14:55:40,128 : INFO : accumulator serialized 2018-03-26 14:55:40,934 : INFO : serializing accumulator to return to master... 2018-03-26 14:55:40,941 : INFO : accumulator serialized 2018-03-26 14:55:41,129 : INFO : 3 accumulators retrieved from output queue 2018-03-26 14:55:41,252 : INFO : accumulated word occurrence stats for 25551537 virtual documents 2018-03-26 14:55:41,571 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_25 2018-03-26 14:55:41,857 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-26 14:55:41,965 : INFO : setting ignored attribute state to None 2018-03-26 14:55:41,965 : INFO : setting ignored attribute dispatcher to None 2018-03-26 14:55:41,966 : INFO : setting ignored attribute id2word to None 2018-03-26 14:55:41,966 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_25 2018-03-26 14:55:41,967 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_25.state 2018-03-26 14:55:42,180 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_25.state 2018-03-26 14:55:42,238 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 14:55:42,350 : INFO : 1 batches submitted to accumulate stats from 64 documents (197179 virtual) 2018-03-26 14:55:42,405 : INFO : 2 batches submitted to accumulate stats from 128 documents (362960 virtual) 2018-03-26 14:55:42,473 : INFO : 3 batches submitted to accumulate stats from 192 documents (520587 virtual) 2018-03-26 14:55:42,651 : INFO : 4 batches submitted to accumulate stats from 256 documents (822479 virtual) 2018-03-26 14:55:42,747 : INFO : 5 batches submitted to accumulate stats from 320 documents (993646 virtual) 2018-03-26 14:55:42,840 : INFO : 6 batches submitted to accumulate stats from 384 documents (1153215 virtual) 2018-03-26 14:55:46,094 : INFO : 7 batches submitted to accumulate stats from 448 documents (1307109 virtual) 2018-03-26 14:55:46,322 : INFO : 8 batches submitted to accumulate stats from 512 documents (1507697 virtual) 2018-03-26 14:55:47,116 : INFO : 9 batches submitted to accumulate stats from 576 documents (1682589 virtual) 2018-03-26 14:55:49,980 : INFO : 10 batches submitted to accumulate stats from 640 documents (1822128 virtual) 2018-03-26 14:55:50,625 : INFO : 11 batches submitted to accumulate stats from 704 documents (2007365 virtual) 2018-03-26 14:55:53,222 : INFO : 12 batches submitted to accumulate stats from 768 documents (2171596 virtual) 2018-03-26 14:55:53,501 : INFO : 13 batches submitted to accumulate stats from 832 documents (2345217 virtual) 2018-03-26 14:55:55,421 : INFO : 14 batches submitted to accumulate stats from 896 documents (2458210 virtual) 2018-03-26 14:55:56,770 : INFO : 15 batches submitted to accumulate stats from 960 documents (2620061 virtual) 2018-03-26 14:55:57,359 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2974369 virtual) 2018-03-26 14:55:59,841 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3166619 virtual) 2018-03-26 14:56:00,621 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3351861 virtual) 2018-03-26 14:56:01,174 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3574250 virtual) 2018-03-26 14:56:02,526 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3761825 virtual) 2018-03-26 14:56:04,470 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3932167 virtual) 2018-03-26 14:56:06,913 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4120818 virtual) 2018-03-26 14:56:08,628 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4939731 virtual) 2018-03-26 14:56:09,436 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5085782 virtual) 2018-03-26 14:56:12,139 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5318482 virtual) 2018-03-26 14:56:12,934 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5447489 virtual) 2018-03-26 14:56:13,298 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5663066 virtual) 2018-03-26 14:56:16,450 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5815377 virtual) 2018-03-26 14:56:16,589 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6048226 virtual) 2018-03-26 14:56:19,375 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6252716 virtual) 2018-03-26 14:56:21,784 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6406543 virtual) 2018-03-26 14:56:24,549 : INFO : 32 batches submitted to accumulate stats from 2048 documents (8011906 virtual) 2018-03-26 14:56:25,519 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8451297 virtual) 2018-03-26 14:56:29,658 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8693702 virtual) 2018-03-26 14:56:30,267 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8859367 virtual) 2018-03-26 14:56:31,547 : INFO : 36 batches submitted to accumulate stats from 2304 documents (9012292 virtual) 2018-03-26 14:56:33,077 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9145938 virtual) 2018-03-26 14:56:38,584 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9340253 virtual) 2018-03-26 14:56:40,138 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9488854 virtual) 2018-03-26 14:56:42,224 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9661822 virtual) 2018-03-26 14:56:43,781 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9828037 virtual) 2018-03-26 14:56:45,178 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9990705 virtual) 2018-03-26 14:56:48,097 : INFO : 43 batches submitted to accumulate stats from 2752 documents (10207716 virtual) 2018-03-26 14:56:48,642 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10384069 virtual) 2018-03-26 14:56:52,122 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10548972 virtual) 2018-03-26 14:56:52,544 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10697445 virtual) 2018-03-26 14:56:55,965 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10844290 virtual) 2018-03-26 14:56:57,316 : INFO : 48 batches submitted to accumulate stats from 3072 documents (11005526 virtual) 2018-03-26 14:57:00,028 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11180984 virtual) 2018-03-26 14:57:01,123 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11394123 virtual) 2018-03-26 14:57:03,296 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11516756 virtual) 2018-03-26 14:57:04,574 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11640485 virtual) 2018-03-26 14:57:05,969 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11843699 virtual) 2018-03-26 14:57:06,930 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12001264 virtual) 2018-03-26 14:57:08,589 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12161397 virtual) 2018-03-26 14:57:09,764 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12336924 virtual) 2018-03-26 14:57:10,807 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12499029 virtual) 2018-03-26 14:57:11,589 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12679012 virtual) 2018-03-26 14:57:14,211 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12807543 virtual) 2018-03-26 14:57:14,556 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12997324 virtual) 2018-03-26 14:57:15,409 : INFO : 61 batches submitted to accumulate stats from 3904 documents (13198288 virtual) 2018-03-26 14:57:18,185 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13326988 virtual) 2018-03-26 14:57:18,273 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13473205 virtual) 2018-03-26 14:57:19,481 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13617515 virtual) 2018-03-26 14:57:21,219 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13751447 virtual) 2018-03-26 14:57:22,380 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13901176 virtual) 2018-03-26 14:57:24,152 : INFO : 67 batches submitted to accumulate stats from 4288 documents (14064198 virtual) 2018-03-26 14:57:24,307 : INFO : 68 batches submitted to accumulate stats from 4352 documents (14237353 virtual) 2018-03-26 14:57:25,728 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14418686 virtual) 2018-03-26 14:57:27,424 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14534833 virtual) 2018-03-26 14:57:27,570 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14786991 virtual) 2018-03-26 14:57:29,240 : INFO : 72 batches submitted to accumulate stats from 4608 documents (15008971 virtual) 2018-03-26 14:57:31,209 : INFO : 73 batches submitted to accumulate stats from 4672 documents (15166558 virtual) 2018-03-26 14:57:31,436 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15339775 virtual) 2018-03-26 14:57:33,312 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15520924 virtual) 2018-03-26 14:57:34,014 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15679380 virtual) 2018-03-26 14:57:36,945 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15803513 virtual) 2018-03-26 14:57:37,631 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15951268 virtual) 2018-03-26 14:57:38,444 : INFO : 79 batches submitted to accumulate stats from 5056 documents (16172642 virtual) 2018-03-26 14:57:40,583 : INFO : 80 batches submitted to accumulate stats from 5120 documents (16287120 virtual) 2018-03-26 14:57:41,927 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16437440 virtual) 2018-03-26 14:57:42,321 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16616702 virtual) 2018-03-26 14:57:43,535 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16818775 virtual) 2018-03-26 14:57:45,378 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16963671 virtual) 2018-03-26 14:57:46,240 : INFO : 85 batches submitted to accumulate stats from 5440 documents (17242931 virtual) 2018-03-26 14:57:47,320 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17417468 virtual) 2018-03-26 14:57:48,994 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17668640 virtual) 2018-03-26 14:57:50,174 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17820744 virtual) 2018-03-26 14:57:52,155 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17945520 virtual) 2018-03-26 14:57:52,255 : INFO : 90 batches submitted to accumulate stats from 5760 documents (18107053 virtual) 2018-03-26 14:57:55,913 : INFO : 91 batches submitted to accumulate stats from 5824 documents (18297035 virtual) 2018-03-26 14:57:56,251 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18445578 virtual) 2018-03-26 14:57:57,817 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18621574 virtual) 2018-03-26 14:57:59,190 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18906839 virtual) 2018-03-26 14:57:59,551 : INFO : 95 batches submitted to accumulate stats from 6080 documents (19043087 virtual) 2018-03-26 14:58:02,475 : INFO : 96 batches submitted to accumulate stats from 6144 documents (19182601 virtual) 2018-03-26 14:58:03,227 : INFO : 97 batches submitted to accumulate stats from 6208 documents (19367218 virtual) 2018-03-26 14:58:04,397 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19486199 virtual) 2018-03-26 14:58:06,621 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19649529 virtual) 2018-03-26 14:58:07,558 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19856105 virtual) 2018-03-26 14:58:09,771 : INFO : 101 batches submitted to accumulate stats from 6464 documents (20061390 virtual) 2018-03-26 14:58:09,955 : INFO : 102 batches submitted to accumulate stats from 6528 documents (20288794 virtual) 2018-03-26 14:58:11,953 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20462255 virtual) 2018-03-26 14:58:12,510 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20600023 virtual) 2018-03-26 14:58:13,712 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20778699 virtual) 2018-03-26 14:58:16,847 : INFO : 106 batches submitted to accumulate stats from 6784 documents (22350807 virtual) 2018-03-26 14:58:17,363 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22474875 virtual) 2018-03-26 14:58:19,198 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22642691 virtual) 2018-03-26 14:58:20,493 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22821796 virtual) 2018-03-26 14:58:21,043 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22964175 virtual) 2018-03-26 14:58:23,464 : INFO : 111 batches submitted to accumulate stats from 7104 documents (23171069 virtual) 2018-03-26 14:58:23,971 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23417327 virtual) 2018-03-26 14:58:27,336 : INFO : 113 batches submitted to accumulate stats from 7232 documents (23675779 virtual) 2018-03-26 14:58:28,215 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23921307 virtual) 2018-03-26 14:58:30,505 : INFO : 115 batches submitted to accumulate stats from 7360 documents (24076034 virtual) 2018-03-26 14:58:32,302 : INFO : 116 batches submitted to accumulate stats from 7424 documents (24265981 virtual) 2018-03-26 14:58:36,055 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24427019 virtual) 2018-03-26 14:58:38,331 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24612536 virtual) 2018-03-26 14:58:41,351 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24824739 virtual) 2018-03-26 14:58:41,964 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24978134 virtual) 2018-03-26 14:58:45,774 : INFO : 121 batches submitted to accumulate stats from 7744 documents (25069581 virtual) 2018-03-26 14:58:45,879 : INFO : 122 batches submitted to accumulate stats from 7808 documents (25243626 virtual) 2018-03-26 14:58:50,089 : INFO : 123 batches submitted to accumulate stats from 7872 documents (25408411 virtual) 2018-03-26 14:58:50,623 : INFO : 124 batches submitted to accumulate stats from 7936 documents (25551102 virtual) 2018-03-26 14:58:56,752 : INFO : serializing accumulator to return to master... 2018-03-26 14:58:56,758 : INFO : accumulator serialized 2018-03-26 14:58:57,275 : INFO : serializing accumulator to return to master... 2018-03-26 14:58:57,280 : INFO : accumulator serialized 2018-03-26 14:58:58,508 : INFO : serializing accumulator to return to master... 2018-03-26 14:58:58,515 : INFO : accumulator serialized 2018-03-26 14:58:58,761 : INFO : 3 accumulators retrieved from output queue 2018-03-26 14:58:58,930 : INFO : accumulated word occurrence stats for 25551422 virtual documents 2018-03-26 14:58:59,314 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_30 2018-03-26 14:58:59,364 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-26 14:58:59,473 : INFO : setting ignored attribute state to None 2018-03-26 14:58:59,474 : INFO : setting ignored attribute dispatcher to None 2018-03-26 14:58:59,474 : INFO : setting ignored attribute id2word to None 2018-03-26 14:58:59,475 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_30 2018-03-26 14:58:59,475 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_30.state 2018-03-26 14:58:59,741 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_30.state 2018-03-26 14:58:59,809 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 14:58:59,922 : INFO : 1 batches submitted to accumulate stats from 64 documents (197185 virtual) 2018-03-26 14:58:59,979 : INFO : 2 batches submitted to accumulate stats from 128 documents (362915 virtual) 2018-03-26 14:59:00,056 : INFO : 3 batches submitted to accumulate stats from 192 documents (517020 virtual) 2018-03-26 14:59:00,239 : INFO : 4 batches submitted to accumulate stats from 256 documents (818899 virtual) 2018-03-26 14:59:00,353 : INFO : 5 batches submitted to accumulate stats from 320 documents (992042 virtual) 2018-03-26 14:59:00,443 : INFO : 6 batches submitted to accumulate stats from 384 documents (1148008 virtual) 2018-03-26 14:59:03,815 : INFO : 7 batches submitted to accumulate stats from 448 documents (1303579 virtual) 2018-03-26 14:59:03,945 : INFO : 8 batches submitted to accumulate stats from 512 documents (1495798 virtual) 2018-03-26 14:59:04,729 : INFO : 9 batches submitted to accumulate stats from 576 documents (1680839 virtual) 2018-03-26 14:59:08,144 : INFO : 10 batches submitted to accumulate stats from 640 documents (1818739 virtual) 2018-03-26 14:59:08,419 : INFO : 11 batches submitted to accumulate stats from 704 documents (1988219 virtual) 2018-03-26 14:59:11,171 : INFO : 12 batches submitted to accumulate stats from 768 documents (2159902 virtual) 2018-03-26 14:59:11,640 : INFO : 13 batches submitted to accumulate stats from 832 documents (2331768 virtual) 2018-03-26 14:59:12,936 : INFO : 14 batches submitted to accumulate stats from 896 documents (2452628 virtual) 2018-03-26 14:59:15,035 : INFO : 15 batches submitted to accumulate stats from 960 documents (2594649 virtual) 2018-03-26 14:59:15,607 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2961373 virtual) 2018-03-26 14:59:17,054 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3154729 virtual) 2018-03-26 14:59:19,111 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3338822 virtual) 2018-03-26 14:59:19,497 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3558812 virtual) 2018-03-26 14:59:20,078 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3750984 virtual) 2018-03-26 14:59:22,548 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3924851 virtual) 2018-03-26 14:59:24,520 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4108285 virtual) 2018-03-26 14:59:26,966 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4933351 virtual) 2018-03-26 14:59:28,558 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5083665 virtual) 2018-03-26 14:59:29,601 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5305424 virtual) 2018-03-26 14:59:31,409 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5447406 virtual) 2018-03-26 14:59:32,626 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5655665 virtual) 2018-03-26 14:59:33,969 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5807741 virtual) 2018-03-26 14:59:36,375 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6037478 virtual) 2018-03-26 14:59:39,073 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6246053 virtual) 2018-03-26 14:59:39,656 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6389035 virtual) 2018-03-26 14:59:43,364 : INFO : 32 batches submitted to accumulate stats from 2048 documents (8002267 virtual) 2018-03-26 14:59:44,257 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8442646 virtual) 2018-03-26 14:59:48,575 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8603771 virtual) 2018-03-26 14:59:49,196 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8836730 virtual) 2018-03-26 14:59:50,207 : INFO : 36 batches submitted to accumulate stats from 2304 documents (9003935 virtual) 2018-03-26 14:59:51,884 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9138828 virtual) 2018-03-26 14:59:55,732 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9321929 virtual) 2018-03-26 15:00:00,066 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9475585 virtual) 2018-03-26 15:00:01,167 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9643615 virtual) 2018-03-26 15:00:03,864 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9816097 virtual) 2018-03-26 15:00:04,470 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9970432 virtual) 2018-03-26 15:00:08,045 : INFO : 43 batches submitted to accumulate stats from 2752 documents (10190788 virtual) 2018-03-26 15:00:08,212 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10370095 virtual) 2018-03-26 15:00:12,005 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10519712 virtual) 2018-03-26 15:00:12,379 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10671468 virtual) 2018-03-26 15:00:15,546 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10817922 virtual) 2018-03-26 15:00:17,514 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10990924 virtual) 2018-03-26 15:00:19,752 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11141883 virtual) 2018-03-26 15:00:21,013 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11355126 virtual) 2018-03-26 15:00:23,321 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11505130 virtual) 2018-03-26 15:00:24,634 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11634694 virtual) 2018-03-26 15:00:25,182 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11820262 virtual) 2018-03-26 15:00:27,398 : INFO : 54 batches submitted to accumulate stats from 3456 documents (11976029 virtual) 2018-03-26 15:00:28,175 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12112701 virtual) 2018-03-26 15:00:30,215 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12298101 virtual) 2018-03-26 15:00:31,039 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12461305 virtual) 2018-03-26 15:00:31,243 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12628558 virtual) 2018-03-26 15:00:34,407 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12774447 virtual) 2018-03-26 15:00:34,605 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12952770 virtual) 2018-03-26 15:00:34,837 : INFO : 61 batches submitted to accumulate stats from 3904 documents (13173432 virtual) 2018-03-26 15:00:38,504 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13309540 virtual) 2018-03-26 15:00:38,755 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13441684 virtual) 2018-03-26 15:00:38,913 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13579868 virtual) 2018-03-26 15:00:42,337 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13725381 virtual) 2018-03-26 15:00:43,284 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13873209 virtual) 2018-03-26 15:00:44,653 : INFO : 67 batches submitted to accumulate stats from 4288 documents (14022676 virtual) 2018-03-26 15:00:45,729 : INFO : 68 batches submitted to accumulate stats from 4352 documents (14219812 virtual) 2018-03-26 15:00:46,725 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14374187 virtual) 2018-03-26 15:00:47,803 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14510690 virtual) 2018-03-26 15:00:49,228 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14734426 virtual) 2018-03-26 15:00:50,357 : INFO : 72 batches submitted to accumulate stats from 4608 documents (14879275 virtual) 2018-03-26 15:00:51,338 : INFO : 73 batches submitted to accumulate stats from 4672 documents (15111523 virtual) 2018-03-26 15:00:53,913 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15295307 virtual) 2018-03-26 15:00:54,025 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15473227 virtual) 2018-03-26 15:00:54,684 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15651393 virtual) 2018-03-26 15:00:57,398 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15772253 virtual) 2018-03-26 15:00:58,841 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15924745 virtual) 2018-03-26 15:01:00,160 : INFO : 79 batches submitted to accumulate stats from 5056 documents (16112931 virtual) 2018-03-26 15:01:01,580 : INFO : 80 batches submitted to accumulate stats from 5120 documents (16266553 virtual) 2018-03-26 15:01:03,085 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16405217 virtual) 2018-03-26 15:01:04,497 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16561315 virtual) 2018-03-26 15:01:04,714 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16755275 virtual) 2018-03-26 15:01:06,753 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16919947 virtual) 2018-03-26 15:01:08,323 : INFO : 85 batches submitted to accumulate stats from 5440 documents (17086906 virtual) 2018-03-26 15:01:08,985 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17360505 virtual) 2018-03-26 15:01:10,143 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17599096 virtual) 2018-03-26 15:01:12,107 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17788873 virtual) 2018-03-26 15:01:13,580 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17917615 virtual) 2018-03-26 15:01:14,038 : INFO : 90 batches submitted to accumulate stats from 5760 documents (18063268 virtual) 2018-03-26 15:01:16,051 : INFO : 91 batches submitted to accumulate stats from 5824 documents (18209246 virtual) 2018-03-26 15:01:19,341 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18409455 virtual) 2018-03-26 15:01:19,504 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18569430 virtual) 2018-03-26 15:01:20,736 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18883302 virtual) 2018-03-26 15:01:22,359 : INFO : 95 batches submitted to accumulate stats from 6080 documents (19003681 virtual) 2018-03-26 15:01:23,096 : INFO : 96 batches submitted to accumulate stats from 6144 documents (19131533 virtual) 2018-03-26 15:01:24,293 : INFO : 97 batches submitted to accumulate stats from 6208 documents (19321572 virtual) 2018-03-26 15:01:26,911 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19443973 virtual) 2018-03-26 15:01:27,156 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19559209 virtual) 2018-03-26 15:01:29,832 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19797936 virtual) 2018-03-26 15:01:30,205 : INFO : 101 batches submitted to accumulate stats from 6464 documents (19995287 virtual) 2018-03-26 15:01:31,668 : INFO : 102 batches submitted to accumulate stats from 6528 documents (20189959 virtual) 2018-03-26 15:01:33,072 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20401764 virtual) 2018-03-26 15:01:34,368 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20555563 virtual) 2018-03-26 15:01:34,491 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20716935 virtual) 2018-03-26 15:01:38,674 : INFO : 106 batches submitted to accumulate stats from 6784 documents (20852741 virtual) 2018-03-26 15:01:39,586 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22437221 virtual) 2018-03-26 15:01:39,681 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22587459 virtual) 2018-03-26 15:01:42,767 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22765289 virtual) 2018-03-26 15:01:43,178 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22911092 virtual) 2018-03-26 15:01:43,943 : INFO : 111 batches submitted to accumulate stats from 7104 documents (23062085 virtual) 2018-03-26 15:01:46,063 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23267860 virtual) 2018-03-26 15:01:47,562 : INFO : 113 batches submitted to accumulate stats from 7232 documents (23581689 virtual) 2018-03-26 15:01:50,331 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23732154 virtual) 2018-03-26 15:01:51,112 : INFO : 115 batches submitted to accumulate stats from 7360 documents (24016952 virtual) 2018-03-26 15:01:53,947 : INFO : 116 batches submitted to accumulate stats from 7424 documents (24189458 virtual) 2018-03-26 15:01:55,421 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24362328 virtual) 2018-03-26 15:01:58,778 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24575119 virtual) 2018-03-26 15:02:01,429 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24711086 virtual) 2018-03-26 15:02:05,093 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24936253 virtual) 2018-03-26 15:02:05,572 : INFO : 121 batches submitted to accumulate stats from 7744 documents (25034620 virtual) 2018-03-26 15:02:09,256 : INFO : 122 batches submitted to accumulate stats from 7808 documents (25170156 virtual) 2018-03-26 15:02:10,556 : INFO : 123 batches submitted to accumulate stats from 7872 documents (25330650 virtual) 2018-03-26 15:02:12,504 : INFO : 124 batches submitted to accumulate stats from 7936 documents (25512587 virtual) 2018-03-26 15:02:14,755 : INFO : 125 batches submitted to accumulate stats from 8000 documents (25551256 virtual) 2018-03-26 15:02:19,181 : INFO : serializing accumulator to return to master... 2018-03-26 15:02:19,187 : INFO : accumulator serialized 2018-03-26 15:02:19,599 : INFO : serializing accumulator to return to master... 2018-03-26 15:02:19,604 : INFO : accumulator serialized 2018-03-26 15:02:21,202 : INFO : serializing accumulator to return to master... 2018-03-26 15:02:21,208 : INFO : accumulator serialized 2018-03-26 15:02:22,345 : INFO : 3 accumulators retrieved from output queue 2018-03-26 15:02:22,580 : INFO : accumulated word occurrence stats for 25551601 virtual documents 2018-03-26 15:02:23,020 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_35 2018-03-26 15:02:23,067 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_35.expElogbeta.npy with mmap=None 2018-03-26 15:02:23,196 : INFO : setting ignored attribute state to None 2018-03-26 15:02:23,196 : INFO : setting ignored attribute dispatcher to None 2018-03-26 15:02:23,197 : INFO : setting ignored attribute id2word to None 2018-03-26 15:02:23,197 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_35 2018-03-26 15:02:23,198 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_35.state 2018-03-26 15:02:23,494 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_35.state 2018-03-26 15:02:23,653 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 15:02:23,767 : INFO : 1 batches submitted to accumulate stats from 64 documents (197185 virtual) 2018-03-26 15:02:23,833 : INFO : 2 batches submitted to accumulate stats from 128 documents (362915 virtual) 2018-03-26 15:02:23,920 : INFO : 3 batches submitted to accumulate stats from 192 documents (517020 virtual) 2018-03-26 15:02:24,108 : INFO : 4 batches submitted to accumulate stats from 256 documents (818899 virtual) 2018-03-26 15:02:24,216 : INFO : 5 batches submitted to accumulate stats from 320 documents (992042 virtual) 2018-03-26 15:02:24,310 : INFO : 6 batches submitted to accumulate stats from 384 documents (1148008 virtual) 2018-03-26 15:02:27,872 : INFO : 7 batches submitted to accumulate stats from 448 documents (1303579 virtual) 2018-03-26 15:02:28,001 : INFO : 8 batches submitted to accumulate stats from 512 documents (1495798 virtual) 2018-03-26 15:02:28,863 : INFO : 9 batches submitted to accumulate stats from 576 documents (1680839 virtual) 2018-03-26 15:02:32,258 : INFO : 10 batches submitted to accumulate stats from 640 documents (1818739 virtual) 2018-03-26 15:02:32,712 : INFO : 11 batches submitted to accumulate stats from 704 documents (1988219 virtual) 2018-03-26 15:02:35,065 : INFO : 12 batches submitted to accumulate stats from 768 documents (2159902 virtual) 2018-03-26 15:02:35,982 : INFO : 13 batches submitted to accumulate stats from 832 documents (2331768 virtual) 2018-03-26 15:02:37,511 : INFO : 14 batches submitted to accumulate stats from 896 documents (2452628 virtual) 2018-03-26 15:02:39,352 : INFO : 15 batches submitted to accumulate stats from 960 documents (2594649 virtual) 2018-03-26 15:02:39,843 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2961373 virtual) 2018-03-26 15:02:41,705 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3154729 virtual) 2018-03-26 15:02:43,656 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3338706 virtual) 2018-03-26 15:02:43,832 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3554439 virtual) 2018-03-26 15:02:44,765 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3746211 virtual) 2018-03-26 15:02:47,397 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3911575 virtual) 2018-03-26 15:02:49,491 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4103718 virtual) 2018-03-26 15:02:52,019 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4929572 virtual) 2018-03-26 15:02:52,776 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5074664 virtual) 2018-03-26 15:02:54,573 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5290593 virtual) 2018-03-26 15:02:56,664 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5444346 virtual) 2018-03-26 15:02:57,009 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5648034 virtual) 2018-03-26 15:02:59,314 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5794377 virtual) 2018-03-26 15:03:00,367 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6035386 virtual) 2018-03-26 15:03:04,145 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6238850 virtual) 2018-03-26 15:03:04,393 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6383697 virtual) 2018-03-26 15:03:08,080 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7994845 virtual) 2018-03-26 15:03:09,351 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8440432 virtual) 2018-03-26 15:03:13,836 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8601355 virtual) 2018-03-26 15:03:14,353 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8834248 virtual) 2018-03-26 15:03:16,466 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8987398 virtual) 2018-03-26 15:03:17,434 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9127912 virtual) 2018-03-26 15:03:22,096 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9301229 virtual) 2018-03-26 15:03:27,195 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9459415 virtual) 2018-03-26 15:03:27,803 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9630777 virtual) 2018-03-26 15:03:30,896 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9809904 virtual) 2018-03-26 15:03:31,101 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9964183 virtual) 2018-03-26 15:03:34,983 : INFO : 43 batches submitted to accumulate stats from 2752 documents (10168878 virtual) 2018-03-26 15:03:35,082 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10344568 virtual) 2018-03-26 15:03:39,145 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10508976 virtual) 2018-03-26 15:03:39,510 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10654720 virtual) 2018-03-26 15:03:42,832 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10787315 virtual) 2018-03-26 15:03:44,348 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10982912 virtual) 2018-03-26 15:03:46,941 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11119440 virtual) 2018-03-26 15:03:48,376 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11346822 virtual) 2018-03-26 15:03:50,387 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11492957 virtual) 2018-03-26 15:03:51,747 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11625426 virtual) 2018-03-26 15:03:53,046 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11795143 virtual) 2018-03-26 15:03:54,933 : INFO : 54 batches submitted to accumulate stats from 3456 documents (11964309 virtual) 2018-03-26 15:03:55,271 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12093789 virtual) 2018-03-26 15:03:58,383 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12278591 virtual) 2018-03-26 15:03:58,486 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12441867 virtual) 2018-03-26 15:03:58,663 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12621963 virtual) 2018-03-26 15:04:01,705 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12759331 virtual) 2018-03-26 15:04:02,595 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12922397 virtual) 2018-03-26 15:04:02,734 : INFO : 61 batches submitted to accumulate stats from 3904 documents (13155554 virtual) 2018-03-26 15:04:06,317 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13291262 virtual) 2018-03-26 15:04:06,638 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13417251 virtual) 2018-03-26 15:04:06,986 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13568703 virtual) 2018-03-26 15:04:09,631 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13703987 virtual) 2018-03-26 15:04:10,614 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13834172 virtual) 2018-03-26 15:04:12,768 : INFO : 67 batches submitted to accumulate stats from 4288 documents (13978931 virtual) 2018-03-26 15:04:12,893 : INFO : 68 batches submitted to accumulate stats from 4352 documents (14179666 virtual) 2018-03-26 15:04:13,752 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14345605 virtual) 2018-03-26 15:04:16,251 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14499576 virtual) 2018-03-26 15:04:16,556 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14708829 virtual) 2018-03-26 15:04:17,088 : INFO : 72 batches submitted to accumulate stats from 4608 documents (14856037 virtual) 2018-03-26 15:04:19,752 : INFO : 73 batches submitted to accumulate stats from 4672 documents (15080012 virtual) 2018-03-26 15:04:21,147 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15264703 virtual) 2018-03-26 15:04:21,510 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15424104 virtual) 2018-03-26 15:04:23,481 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15586947 virtual) 2018-03-26 15:04:25,090 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15742355 virtual) 2018-03-26 15:04:26,074 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15895904 virtual) 2018-03-26 15:04:28,914 : INFO : 79 batches submitted to accumulate stats from 5056 documents (16067361 virtual) 2018-03-26 15:04:29,532 : INFO : 80 batches submitted to accumulate stats from 5120 documents (16246931 virtual) 2018-03-26 15:04:30,001 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16383751 virtual) 2018-03-26 15:04:32,964 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16530128 virtual) 2018-03-26 15:04:33,328 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16714976 virtual) 2018-03-26 15:04:33,903 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16899502 virtual) 2018-03-26 15:04:37,137 : INFO : 85 batches submitted to accumulate stats from 5440 documents (17054656 virtual) 2018-03-26 15:04:37,355 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17309811 virtual) 2018-03-26 15:04:37,885 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17476654 virtual) 2018-03-26 15:04:40,849 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17736596 virtual) 2018-03-26 15:04:41,850 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17874860 virtual) 2018-03-26 15:04:42,345 : INFO : 90 batches submitted to accumulate stats from 5760 documents (18022471 virtual) 2018-03-26 15:04:44,558 : INFO : 91 batches submitted to accumulate stats from 5824 documents (18180010 virtual) 2018-03-26 15:04:46,440 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18357946 virtual) 2018-03-26 15:04:47,543 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18533114 virtual) 2018-03-26 15:04:49,791 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18812149 virtual) 2018-03-26 15:04:50,793 : INFO : 95 batches submitted to accumulate stats from 6080 documents (18965818 virtual) 2018-03-26 15:04:51,373 : INFO : 96 batches submitted to accumulate stats from 6144 documents (19097137 virtual) 2018-03-26 15:04:53,898 : INFO : 97 batches submitted to accumulate stats from 6208 documents (19260684 virtual) 2018-03-26 15:04:55,377 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19409522 virtual) 2018-03-26 15:04:55,780 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19538574 virtual) 2018-03-26 15:04:59,181 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19733327 virtual) 2018-03-26 15:04:59,300 : INFO : 101 batches submitted to accumulate stats from 6464 documents (19928441 virtual) 2018-03-26 15:05:00,479 : INFO : 102 batches submitted to accumulate stats from 6528 documents (20152652 virtual) 2018-03-26 15:05:02,941 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20355016 virtual) 2018-03-26 15:05:03,359 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20521806 virtual) 2018-03-26 15:05:03,764 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20680233 virtual) 2018-03-26 15:05:07,540 : INFO : 106 batches submitted to accumulate stats from 6784 documents (20818595 virtual) 2018-03-26 15:05:08,435 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22415151 virtual) 2018-03-26 15:05:09,407 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22525052 virtual) 2018-03-26 15:05:12,514 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22709560 virtual) 2018-03-26 15:05:12,806 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22879966 virtual) 2018-03-26 15:05:13,235 : INFO : 111 batches submitted to accumulate stats from 7104 documents (23019450 virtual) 2018-03-26 15:05:15,926 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23219545 virtual) 2018-03-26 15:05:16,111 : INFO : 113 batches submitted to accumulate stats from 7232 documents (23520532 virtual) 2018-03-26 15:05:20,195 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23700860 virtual) 2018-03-26 15:05:20,383 : INFO : 115 batches submitted to accumulate stats from 7360 documents (23972854 virtual) 2018-03-26 15:05:23,614 : INFO : 116 batches submitted to accumulate stats from 7424 documents (24125455 virtual) 2018-03-26 15:05:24,566 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24322261 virtual) 2018-03-26 15:05:28,992 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24507532 virtual) 2018-03-26 15:05:30,954 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24680300 virtual) 2018-03-26 15:05:34,604 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24893668 virtual) 2018-03-26 15:05:35,362 : INFO : 121 batches submitted to accumulate stats from 7744 documents (24995758 virtual) 2018-03-26 15:05:39,341 : INFO : 122 batches submitted to accumulate stats from 7808 documents (25118479 virtual) 2018-03-26 15:05:39,794 : INFO : 123 batches submitted to accumulate stats from 7872 documents (25296625 virtual) 2018-03-26 15:05:43,410 : INFO : 124 batches submitted to accumulate stats from 7936 documents (25449426 virtual) 2018-03-26 15:05:44,817 : INFO : 125 batches submitted to accumulate stats from 8000 documents (25551342 virtual) 2018-03-26 15:05:50,352 : INFO : serializing accumulator to return to master... 2018-03-26 15:05:50,358 : INFO : accumulator serialized 2018-03-26 15:05:51,257 : INFO : serializing accumulator to return to master... 2018-03-26 15:05:51,263 : INFO : accumulator serialized 2018-03-26 15:05:52,103 : INFO : serializing accumulator to return to master... 2018-03-26 15:05:52,108 : INFO : accumulator serialized 2018-03-26 15:05:52,416 : INFO : 3 accumulators retrieved from output queue 2018-03-26 15:05:52,720 : INFO : accumulated word occurrence stats for 25551718 virtual documents 2018-03-26 15:05:53,250 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_40 2018-03-26 15:05:53,296 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-26 15:05:53,476 : INFO : setting ignored attribute state to None 2018-03-26 15:05:53,477 : INFO : setting ignored attribute dispatcher to None 2018-03-26 15:05:53,477 : INFO : setting ignored attribute id2word to None 2018-03-26 15:05:53,478 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_40 2018-03-26 15:05:53,479 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_40.state 2018-03-26 15:05:53,863 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_40.state 2018-03-26 15:05:53,948 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 15:05:54,074 : INFO : 1 batches submitted to accumulate stats from 64 documents (197185 virtual) 2018-03-26 15:05:54,133 : INFO : 2 batches submitted to accumulate stats from 128 documents (362915 virtual) 2018-03-26 15:05:54,203 : INFO : 3 batches submitted to accumulate stats from 192 documents (517020 virtual) 2018-03-26 15:05:54,400 : INFO : 4 batches submitted to accumulate stats from 256 documents (818899 virtual) 2018-03-26 15:05:54,507 : INFO : 5 batches submitted to accumulate stats from 320 documents (992042 virtual) 2018-03-26 15:05:54,599 : INFO : 6 batches submitted to accumulate stats from 384 documents (1153098 virtual) 2018-03-26 15:05:58,282 : INFO : 7 batches submitted to accumulate stats from 448 documents (1307015 virtual) 2018-03-26 15:05:58,570 : INFO : 8 batches submitted to accumulate stats from 512 documents (1495827 virtual) 2018-03-26 15:05:59,199 : INFO : 9 batches submitted to accumulate stats from 576 documents (1680936 virtual) 2018-03-26 15:06:03,377 : INFO : 10 batches submitted to accumulate stats from 640 documents (1818795 virtual) 2018-03-26 15:06:03,566 : INFO : 11 batches submitted to accumulate stats from 704 documents (1997044 virtual) 2018-03-26 15:06:06,492 : INFO : 12 batches submitted to accumulate stats from 768 documents (2167325 virtual) 2018-03-26 15:06:07,340 : INFO : 13 batches submitted to accumulate stats from 832 documents (2336675 virtual) 2018-03-26 15:06:08,412 : INFO : 14 batches submitted to accumulate stats from 896 documents (2455038 virtual) 2018-03-26 15:06:11,036 : INFO : 15 batches submitted to accumulate stats from 960 documents (2597171 virtual) 2018-03-26 15:06:11,254 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2966138 virtual) 2018-03-26 15:06:12,933 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3154744 virtual) 2018-03-26 15:06:15,230 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3338929 virtual) 2018-03-26 15:06:15,374 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3568693 virtual) 2018-03-26 15:06:16,016 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3750983 virtual) 2018-03-26 15:06:18,862 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3924850 virtual) 2018-03-26 15:06:20,637 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4108284 virtual) 2018-03-26 15:06:23,604 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4933350 virtual) 2018-03-26 15:06:24,699 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5083664 virtual) 2018-03-26 15:06:26,325 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5305423 virtual) 2018-03-26 15:06:28,316 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5444364 virtual) 2018-03-26 15:06:29,122 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5648101 virtual) 2018-03-26 15:06:30,766 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5800334 virtual) 2018-03-26 15:06:32,776 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6035465 virtual) 2018-03-26 15:06:36,126 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6241632 virtual) 2018-03-26 15:06:36,209 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6387021 virtual) 2018-03-26 15:06:40,048 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7994908 virtual) 2018-03-26 15:06:41,637 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8440443 virtual) 2018-03-26 15:06:45,808 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8603552 virtual) 2018-03-26 15:06:46,704 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8834324 virtual) 2018-03-26 15:06:48,722 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8995531 virtual) 2018-03-26 15:06:49,497 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9128128 virtual) 2018-03-26 15:06:53,638 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9316937 virtual) 2018-03-26 15:06:59,173 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9459508 virtual) 2018-03-26 15:06:59,277 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9631852 virtual) 2018-03-26 15:07:02,386 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9812272 virtual) 2018-03-26 15:07:03,139 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9966838 virtual) 2018-03-26 15:07:06,507 : INFO : 43 batches submitted to accumulate stats from 2752 documents (10173420 virtual) 2018-03-26 15:07:07,143 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10360996 virtual) 2018-03-26 15:07:10,984 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10509180 virtual) 2018-03-26 15:07:11,728 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10654935 virtual) 2018-03-26 15:07:14,837 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10795081 virtual) 2018-03-26 15:07:16,758 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10983050 virtual) 2018-03-26 15:07:19,579 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11125204 virtual) 2018-03-26 15:07:20,472 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11347041 virtual) 2018-03-26 15:07:23,188 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11495983 virtual) 2018-03-26 15:07:24,072 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11625670 virtual) 2018-03-26 15:07:25,235 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11795182 virtual) 2018-03-26 15:07:27,589 : INFO : 54 batches submitted to accumulate stats from 3456 documents (11970074 virtual) 2018-03-26 15:07:27,832 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12101552 virtual) 2018-03-26 15:07:30,742 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12282919 virtual) 2018-03-26 15:07:31,021 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12443707 virtual) 2018-03-26 15:07:31,431 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12623005 virtual) 2018-03-26 15:07:34,734 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12762968 virtual) 2018-03-26 15:07:35,039 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12928833 virtual) 2018-03-26 15:07:35,516 : INFO : 61 batches submitted to accumulate stats from 3904 documents (13156504 virtual) 2018-03-26 15:07:39,152 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13292187 virtual) 2018-03-26 15:07:39,324 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13420339 virtual) 2018-03-26 15:07:40,032 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13571269 virtual) 2018-03-26 15:07:42,634 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13710733 virtual) 2018-03-26 15:07:43,565 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13855020 virtual) 2018-03-26 15:07:45,601 : INFO : 67 batches submitted to accumulate stats from 4288 documents (13986368 virtual) 2018-03-26 15:07:46,121 : INFO : 68 batches submitted to accumulate stats from 4352 documents (14191309 virtual) 2018-03-26 15:07:46,876 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14345969 virtual) 2018-03-26 15:07:49,447 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14499803 virtual) 2018-03-26 15:07:49,678 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14710725 virtual) 2018-03-26 15:07:50,669 : INFO : 72 batches submitted to accumulate stats from 4608 documents (14860052 virtual) 2018-03-26 15:07:52,836 : INFO : 73 batches submitted to accumulate stats from 4672 documents (15083751 virtual) 2018-03-26 15:07:54,585 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15267838 virtual) 2018-03-26 15:07:54,883 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15435513 virtual) 2018-03-26 15:07:56,813 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15603628 virtual) 2018-03-26 15:07:58,514 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15752131 virtual) 2018-03-26 15:07:59,652 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15902912 virtual) 2018-03-26 15:08:02,333 : INFO : 79 batches submitted to accumulate stats from 5056 documents (16073394 virtual) 2018-03-26 15:08:02,940 : INFO : 80 batches submitted to accumulate stats from 5120 documents (16247327 virtual) 2018-03-26 15:08:04,214 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16387680 virtual) 2018-03-26 15:08:06,669 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16532816 virtual) 2018-03-26 15:08:06,780 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16721341 virtual) 2018-03-26 15:08:08,002 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16903970 virtual) 2018-03-26 15:08:11,029 : INFO : 85 batches submitted to accumulate stats from 5440 documents (17054752 virtual) 2018-03-26 15:08:11,326 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17313182 virtual) 2018-03-26 15:08:11,581 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17497031 virtual) 2018-03-26 15:08:14,781 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17756251 virtual) 2018-03-26 15:08:16,083 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17882281 virtual) 2018-03-26 15:08:16,250 : INFO : 90 batches submitted to accumulate stats from 5760 documents (18030235 virtual) 2018-03-26 15:08:18,442 : INFO : 91 batches submitted to accumulate stats from 5824 documents (18183334 virtual) 2018-03-26 15:08:20,610 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18358148 virtual) 2018-03-26 15:08:22,342 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18537192 virtual) 2018-03-26 15:08:23,748 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18825075 virtual) 2018-03-26 15:08:24,904 : INFO : 95 batches submitted to accumulate stats from 6080 documents (18974238 virtual) 2018-03-26 15:08:26,085 : INFO : 96 batches submitted to accumulate stats from 6144 documents (19108625 virtual) 2018-03-26 15:08:27,715 : INFO : 97 batches submitted to accumulate stats from 6208 documents (19271087 virtual) 2018-03-26 15:08:29,333 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19409681 virtual) 2018-03-26 15:08:30,776 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19538734 virtual) 2018-03-26 15:08:33,015 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19738500 virtual) 2018-03-26 15:08:34,151 : INFO : 101 batches submitted to accumulate stats from 6464 documents (19928566 virtual) 2018-03-26 15:08:35,044 : INFO : 102 batches submitted to accumulate stats from 6528 documents (20152773 virtual) 2018-03-26 15:08:37,365 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20355194 virtual) 2018-03-26 15:08:37,647 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20522007 virtual) 2018-03-26 15:08:38,447 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20680305 virtual) 2018-03-26 15:08:43,008 : INFO : 106 batches submitted to accumulate stats from 6784 documents (20818626 virtual) 2018-03-26 15:08:43,930 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22415254 virtual) 2018-03-26 15:08:44,922 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22525143 virtual) 2018-03-26 15:08:47,683 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22715405 virtual) 2018-03-26 15:08:48,492 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22880298 virtual) 2018-03-26 15:08:48,890 : INFO : 111 batches submitted to accumulate stats from 7104 documents (23019449 virtual) 2018-03-26 15:08:51,156 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23219678 virtual) 2018-03-26 15:08:51,733 : INFO : 113 batches submitted to accumulate stats from 7232 documents (23521598 virtual) 2018-03-26 15:08:55,802 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23701020 virtual) 2018-03-26 15:08:56,020 : INFO : 115 batches submitted to accumulate stats from 7360 documents (23972959 virtual) 2018-03-26 15:08:59,350 : INFO : 116 batches submitted to accumulate stats from 7424 documents (24126673 virtual) 2018-03-26 15:09:00,386 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24322339 virtual) 2018-03-26 15:09:04,741 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24517930 virtual) 2018-03-26 15:09:06,708 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24682945 virtual) 2018-03-26 15:09:10,705 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24893766 virtual) 2018-03-26 15:09:11,291 : INFO : 121 batches submitted to accumulate stats from 7744 documents (24996989 virtual) 2018-03-26 15:09:15,602 : INFO : 122 batches submitted to accumulate stats from 7808 documents (25118549 virtual) 2018-03-26 15:09:16,120 : INFO : 123 batches submitted to accumulate stats from 7872 documents (25296642 virtual) 2018-03-26 15:09:19,627 : INFO : 124 batches submitted to accumulate stats from 7936 documents (25449443 virtual) 2018-03-26 15:09:21,455 : INFO : 125 batches submitted to accumulate stats from 8000 documents (25551359 virtual) 2018-03-26 15:09:26,750 : INFO : serializing accumulator to return to master... 2018-03-26 15:09:26,831 : INFO : accumulator serialized 2018-03-26 15:09:27,913 : INFO : serializing accumulator to return to master... 2018-03-26 15:09:27,918 : INFO : accumulator serialized 2018-03-26 15:09:28,348 : INFO : serializing accumulator to return to master... 2018-03-26 15:09:28,354 : INFO : accumulator serialized 2018-03-26 15:09:28,708 : INFO : 3 accumulators retrieved from output queue 2018-03-26 15:09:29,088 : INFO : accumulated word occurrence stats for 25551734 virtual documents 2018-03-26 15:09:29,645 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_50 2018-03-26 15:09:29,701 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_50.expElogbeta.npy with mmap=None 2018-03-26 15:09:29,907 : INFO : setting ignored attribute state to None 2018-03-26 15:09:29,908 : INFO : setting ignored attribute dispatcher to None 2018-03-26 15:09:29,908 : INFO : setting ignored attribute id2word to None 2018-03-26 15:09:29,909 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_50 2018-03-26 15:09:29,909 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_50.state 2018-03-26 15:09:30,329 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_50.state 2018-03-26 15:09:30,425 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 15:09:30,541 : INFO : 1 batches submitted to accumulate stats from 64 documents (197185 virtual) 2018-03-26 15:09:30,601 : INFO : 2 batches submitted to accumulate stats from 128 documents (362915 virtual) 2018-03-26 15:09:30,670 : INFO : 3 batches submitted to accumulate stats from 192 documents (517020 virtual) 2018-03-26 15:09:30,872 : INFO : 4 batches submitted to accumulate stats from 256 documents (818899 virtual) 2018-03-26 15:09:30,975 : INFO : 5 batches submitted to accumulate stats from 320 documents (992042 virtual) 2018-03-26 15:09:31,070 : INFO : 6 batches submitted to accumulate stats from 384 documents (1148008 virtual) 2018-03-26 15:09:34,825 : INFO : 7 batches submitted to accumulate stats from 448 documents (1303579 virtual) 2018-03-26 15:09:34,997 : INFO : 8 batches submitted to accumulate stats from 512 documents (1495798 virtual) 2018-03-26 15:09:35,909 : INFO : 9 batches submitted to accumulate stats from 576 documents (1680839 virtual) 2018-03-26 15:09:39,523 : INFO : 10 batches submitted to accumulate stats from 640 documents (1818739 virtual) 2018-03-26 15:09:40,016 : INFO : 11 batches submitted to accumulate stats from 704 documents (1988219 virtual) 2018-03-26 15:09:42,636 : INFO : 12 batches submitted to accumulate stats from 768 documents (2159902 virtual) 2018-03-26 15:09:43,380 : INFO : 13 batches submitted to accumulate stats from 832 documents (2331768 virtual) 2018-03-26 15:09:45,065 : INFO : 14 batches submitted to accumulate stats from 896 documents (2452628 virtual) 2018-03-26 15:09:46,898 : INFO : 15 batches submitted to accumulate stats from 960 documents (2594649 virtual) 2018-03-26 15:09:47,597 : INFO : 16 batches submitted to accumulate stats from 1024 documents (2961373 virtual) 2018-03-26 15:09:49,531 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3154729 virtual) 2018-03-26 15:09:51,622 : INFO : 18 batches submitted to accumulate stats from 1152 documents (3338706 virtual) 2018-03-26 15:09:51,740 : INFO : 19 batches submitted to accumulate stats from 1216 documents (3554439 virtual) 2018-03-26 15:09:52,804 : INFO : 20 batches submitted to accumulate stats from 1280 documents (3746307 virtual) 2018-03-26 15:09:55,633 : INFO : 21 batches submitted to accumulate stats from 1344 documents (3914036 virtual) 2018-03-26 15:09:57,739 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4108274 virtual) 2018-03-26 15:10:00,604 : INFO : 23 batches submitted to accumulate stats from 1472 documents (4933273 virtual) 2018-03-26 15:10:00,891 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5083560 virtual) 2018-03-26 15:10:03,253 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5301515 virtual) 2018-03-26 15:10:05,208 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5444357 virtual) 2018-03-26 15:10:05,649 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5648045 virtual) 2018-03-26 15:10:08,311 : INFO : 28 batches submitted to accumulate stats from 1792 documents (5794388 virtual) 2018-03-26 15:10:09,505 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6035397 virtual) 2018-03-26 15:10:13,157 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6238861 virtual) 2018-03-26 15:10:13,871 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6383708 virtual) 2018-03-26 15:10:17,869 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7994856 virtual) 2018-03-26 15:10:18,568 : INFO : 33 batches submitted to accumulate stats from 2112 documents (8440317 virtual) 2018-03-26 15:10:23,638 : INFO : 34 batches submitted to accumulate stats from 2176 documents (8601377 virtual) 2018-03-26 15:10:24,033 : INFO : 35 batches submitted to accumulate stats from 2240 documents (8834270 virtual) 2018-03-26 15:10:25,745 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8987420 virtual) 2018-03-26 15:10:27,411 : INFO : 37 batches submitted to accumulate stats from 2368 documents (9128042 virtual) 2018-03-26 15:10:31,669 : INFO : 38 batches submitted to accumulate stats from 2432 documents (9315588 virtual) 2018-03-26 15:10:35,901 : INFO : 39 batches submitted to accumulate stats from 2496 documents (9459520 virtual) 2018-03-26 15:10:37,716 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9631709 virtual) 2018-03-26 15:10:39,912 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9809922 virtual) 2018-03-26 15:10:41,232 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9964201 virtual) 2018-03-26 15:10:44,639 : INFO : 43 batches submitted to accumulate stats from 2752 documents (10168896 virtual) 2018-03-26 15:10:44,957 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10344586 virtual) 2018-03-26 15:10:49,136 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10508994 virtual) 2018-03-26 15:10:49,751 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10654738 virtual) 2018-03-26 15:10:53,071 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10787333 virtual) 2018-03-26 15:10:54,885 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10982930 virtual) 2018-03-26 15:10:57,567 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11119458 virtual) 2018-03-26 15:10:59,187 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11337078 virtual) 2018-03-26 15:11:01,386 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11492850 virtual) 2018-03-26 15:11:02,578 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11625402 virtual) 2018-03-26 15:11:02,670 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11792633 virtual) 2018-03-26 15:11:06,035 : INFO : 54 batches submitted to accumulate stats from 3456 documents (11964334 virtual) 2018-03-26 15:11:06,476 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12093775 virtual) 2018-03-26 15:11:08,199 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12270682 virtual) 2018-03-26 15:11:09,840 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12440606 virtual) 2018-03-26 15:11:10,185 : INFO : 58 batches submitted to accumulate stats from 3712 documents (12619087 virtual) 2018-03-26 15:11:12,407 : INFO : 59 batches submitted to accumulate stats from 3776 documents (12759358 virtual) 2018-03-26 15:11:13,594 : INFO : 60 batches submitted to accumulate stats from 3840 documents (12922424 virtual) 2018-03-26 15:11:14,403 : INFO : 61 batches submitted to accumulate stats from 3904 documents (13155689 virtual) 2018-03-26 15:11:17,121 : INFO : 62 batches submitted to accumulate stats from 3968 documents (13291467 virtual) 2018-03-26 15:11:18,162 : INFO : 63 batches submitted to accumulate stats from 4032 documents (13417423 virtual) 2018-03-26 15:11:18,935 : INFO : 64 batches submitted to accumulate stats from 4096 documents (13571286 virtual) 2018-03-26 15:11:20,884 : INFO : 65 batches submitted to accumulate stats from 4160 documents (13710694 virtual) 2018-03-26 15:11:22,854 : INFO : 66 batches submitted to accumulate stats from 4224 documents (13854984 virtual) 2018-03-26 15:11:24,858 : INFO : 67 batches submitted to accumulate stats from 4288 documents (13978952 virtual) 2018-03-26 15:11:25,334 : INFO : 68 batches submitted to accumulate stats from 4352 documents (14179687 virtual) 2018-03-26 15:11:26,311 : INFO : 69 batches submitted to accumulate stats from 4416 documents (14345626 virtual) 2018-03-26 15:11:28,940 : INFO : 70 batches submitted to accumulate stats from 4480 documents (14499597 virtual) 2018-03-26 15:11:29,064 : INFO : 71 batches submitted to accumulate stats from 4544 documents (14708850 virtual) 2018-03-26 15:11:30,106 : INFO : 72 batches submitted to accumulate stats from 4608 documents (14856058 virtual) 2018-03-26 15:11:32,262 : INFO : 73 batches submitted to accumulate stats from 4672 documents (15080125 virtual) 2018-03-26 15:11:34,341 : INFO : 74 batches submitted to accumulate stats from 4736 documents (15264728 virtual) 2018-03-26 15:11:34,435 : INFO : 75 batches submitted to accumulate stats from 4800 documents (15424261 virtual) 2018-03-26 15:11:36,290 : INFO : 76 batches submitted to accumulate stats from 4864 documents (15593849 virtual) 2018-03-26 15:11:38,086 : INFO : 77 batches submitted to accumulate stats from 4928 documents (15746917 virtual) 2018-03-26 15:11:39,461 : INFO : 78 batches submitted to accumulate stats from 4992 documents (15896167 virtual) 2018-03-26 15:11:42,031 : INFO : 79 batches submitted to accumulate stats from 5056 documents (16067468 virtual) 2018-03-26 15:11:42,814 : INFO : 80 batches submitted to accumulate stats from 5120 documents (16246998 virtual) 2018-03-26 15:11:43,730 : INFO : 81 batches submitted to accumulate stats from 5184 documents (16387498 virtual) 2018-03-26 15:11:46,552 : INFO : 82 batches submitted to accumulate stats from 5248 documents (16532736 virtual) 2018-03-26 15:11:46,849 : INFO : 83 batches submitted to accumulate stats from 5312 documents (16715085 virtual) 2018-03-26 15:11:47,786 : INFO : 84 batches submitted to accumulate stats from 5376 documents (16903926 virtual) 2018-03-26 15:11:50,956 : INFO : 85 batches submitted to accumulate stats from 5440 documents (17054713 virtual) 2018-03-26 15:11:51,569 : INFO : 86 batches submitted to accumulate stats from 5504 documents (17309907 virtual) 2018-03-26 15:11:51,741 : INFO : 87 batches submitted to accumulate stats from 5568 documents (17492583 virtual) 2018-03-26 15:11:54,898 : INFO : 88 batches submitted to accumulate stats from 5632 documents (17741016 virtual) 2018-03-26 15:11:56,401 : INFO : 89 batches submitted to accumulate stats from 5696 documents (17876354 virtual) 2018-03-26 15:11:56,497 : INFO : 90 batches submitted to accumulate stats from 5760 documents (18024534 virtual) 2018-03-26 15:11:58,880 : INFO : 91 batches submitted to accumulate stats from 5824 documents (18183217 virtual) 2018-03-26 15:12:00,956 : INFO : 92 batches submitted to accumulate stats from 5888 documents (18358028 virtual) 2018-03-26 15:12:02,403 : INFO : 93 batches submitted to accumulate stats from 5952 documents (18533288 virtual) 2018-03-26 15:12:04,479 : INFO : 94 batches submitted to accumulate stats from 6016 documents (18823897 virtual) 2018-03-26 15:12:05,331 : INFO : 95 batches submitted to accumulate stats from 6080 documents (18969878 virtual) 2018-03-26 15:12:06,301 : INFO : 96 batches submitted to accumulate stats from 6144 documents (19104108 virtual) 2018-03-26 15:12:08,671 : INFO : 97 batches submitted to accumulate stats from 6208 documents (19260766 virtual) 2018-03-26 15:12:09,890 : INFO : 98 batches submitted to accumulate stats from 6272 documents (19409615 virtual) 2018-03-26 15:12:11,017 : INFO : 99 batches submitted to accumulate stats from 6336 documents (19538592 virtual) 2018-03-26 15:12:13,723 : INFO : 100 batches submitted to accumulate stats from 6400 documents (19733345 virtual) 2018-03-26 15:12:14,522 : INFO : 101 batches submitted to accumulate stats from 6464 documents (19928459 virtual) 2018-03-26 15:12:16,152 : INFO : 102 batches submitted to accumulate stats from 6528 documents (20152670 virtual) 2018-03-26 15:12:17,791 : INFO : 103 batches submitted to accumulate stats from 6592 documents (20355034 virtual) 2018-03-26 15:12:18,435 : INFO : 104 batches submitted to accumulate stats from 6656 documents (20521824 virtual) 2018-03-26 15:12:19,504 : INFO : 105 batches submitted to accumulate stats from 6720 documents (20680251 virtual) 2018-03-26 15:12:23,100 : INFO : 106 batches submitted to accumulate stats from 6784 documents (20818613 virtual) 2018-03-26 15:12:23,990 : INFO : 107 batches submitted to accumulate stats from 6848 documents (22415169 virtual) 2018-03-26 15:12:25,399 : INFO : 108 batches submitted to accumulate stats from 6912 documents (22525070 virtual) 2018-03-26 15:12:27,983 : INFO : 109 batches submitted to accumulate stats from 6976 documents (22706421 virtual) 2018-03-26 15:12:28,616 : INFO : 110 batches submitted to accumulate stats from 7040 documents (22879970 virtual) 2018-03-26 15:12:29,593 : INFO : 111 batches submitted to accumulate stats from 7104 documents (23019004 virtual) 2018-03-26 15:12:31,592 : INFO : 112 batches submitted to accumulate stats from 7168 documents (23219317 virtual) 2018-03-26 15:12:32,552 : INFO : 113 batches submitted to accumulate stats from 7232 documents (23514833 virtual) 2018-03-26 15:12:36,233 : INFO : 114 batches submitted to accumulate stats from 7296 documents (23700700 virtual) 2018-03-26 15:12:37,097 : INFO : 115 batches submitted to accumulate stats from 7360 documents (23957550 virtual) 2018-03-26 15:12:40,015 : INFO : 116 batches submitted to accumulate stats from 7424 documents (24113385 virtual) 2018-03-26 15:12:41,569 : INFO : 117 batches submitted to accumulate stats from 7488 documents (24309477 virtual) 2018-03-26 15:12:46,164 : INFO : 118 batches submitted to accumulate stats from 7552 documents (24507498 virtual) 2018-03-26 15:12:47,747 : INFO : 119 batches submitted to accumulate stats from 7616 documents (24680257 virtual) 2018-03-26 15:12:51,965 : INFO : 120 batches submitted to accumulate stats from 7680 documents (24890878 virtual) 2018-03-26 15:12:52,440 : INFO : 121 batches submitted to accumulate stats from 7744 documents (24995774 virtual) 2018-03-26 15:12:56,930 : INFO : 122 batches submitted to accumulate stats from 7808 documents (25118495 virtual) 2018-03-26 15:12:57,567 : INFO : 123 batches submitted to accumulate stats from 7872 documents (25293629 virtual) 2018-03-26 15:13:01,079 : INFO : 124 batches submitted to accumulate stats from 7936 documents (25435348 virtual) 2018-03-26 15:13:03,139 : INFO : 125 batches submitted to accumulate stats from 8000 documents (25551374 virtual) 2018-03-26 15:13:08,517 : INFO : serializing accumulator to return to master... 2018-03-26 15:13:08,614 : INFO : accumulator serialized 2018-03-26 15:13:09,546 : INFO : serializing accumulator to return to master... 2018-03-26 15:13:09,683 : INFO : serializing accumulator to return to master... 2018-03-26 15:13:09,551 : INFO : accumulator serialized 2018-03-26 15:13:09,689 : INFO : accumulator serialized 2018-03-26 15:13:10,169 : INFO : 3 accumulators retrieved from output queue 2018-03-26 15:13:10,654 : INFO : accumulated word occurrence stats for 25551753 virtual documents
Best number of topics¶
In [28]:
tmfull_score
Out[28]:
| num_topic | c_v | u_mass | c_npmi | c_uci | |
|---|---|---|---|---|---|
| 0 | 10 | 0.518523 | -0.786897 | 0.039421 | 0.177287 |
| 1 | 15 | 0.566641 | -0.799935 | 0.060995 | 0.411027 |
| 2 | 20 | 0.540166 | -0.881306 | 0.043719 | 0.022262 |
| 3 | 25 | 0.554963 | -0.791286 | 0.058475 | 0.371627 |
| 4 | 30 | 0.565710 | -0.951031 | 0.063559 | 0.345635 |
| 5 | 35 | 0.567563 | -0.931557 | 0.065134 | 0.382193 |
| 6 | 40 | 0.574072 | -0.939914 | 0.068184 | 0.413204 |
| 7 | 50 | 0.564449 | -1.052651 | 0.063672 | 0.309491 |
In [29]:
f = plt.figure(figsize=(8,8))
ax1 = f.add_subplot(211)
ax1 = tmfull_score.plot(x='num_topic',y='c_v',ax=ax1)
ax1 = tmfull_score.plot(x='num_topic', y='u_mass', ax=ax1, secondary_y=True)
ax2 = f.add_subplot(212)
ax2 = tmfull_score.plot(x='num_topic', y='c_npmi', ax=ax2)
ax2 = tmfull_score.plot(x='num_topic', y='c_uci',ax=ax2, secondary_y=True)
Top terms for LDA¶
In [33]:
best_num_topics = 25
tmfull_best_model = models.LdaModel.load(
'../src/tmfull/ldamodel_nb_topics_{}'.format(best_num_topics))
2018-03-26 15:57:53,908 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_25 2018-03-26 15:57:53,914 : INFO : loading expElogbeta from ../src/tmfull/ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-26 15:57:53,923 : INFO : setting ignored attribute state to None 2018-03-26 15:57:53,923 : INFO : setting ignored attribute dispatcher to None 2018-03-26 15:57:53,924 : INFO : setting ignored attribute id2word to None 2018-03-26 15:57:53,924 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_25 2018-03-26 15:57:53,925 : INFO : loading LdaModel object from ../src/tmfull/ldamodel_nb_topics_25.state 2018-03-26 15:57:54,006 : INFO : loaded ../src/tmfull/ldamodel_nb_topics_25.state
In [35]:
for i in range(best_num_topics):
print('TOPIC {}'.format(i))
eg = tmfull_best_model.get_topic_terms(i, topn=15)
for (a,b) in eg:
print('{}\t\t{:.5f}'.format(dict_tmfull[a], b))
print('\n\n')
TOPIC 0 type 0.01991 language 0.00861 program 0.00844 function 0.00776 we 0.00689 code 0.00621 programming 0.00618 value 0.00585 example 0.00457 case 0.00439 semantics 0.00400 rule 0.00380 implementation 0.00378 name 0.00363 operation 0.00354 TOPIC 1 performance 0.00815 memory 0.00797 cache 0.00643 network 0.00614 core 0.00544 instruction 0.00540 application 0.00475 we 0.00436 number 0.00391 execution 0.00378 processor 0.00370 block 0.00365 figure 0.00359 size 0.00354 architecture 0.00343 TOPIC 2 service 0.00658 information 0.00594 web 0.00445 application 0.00440 tool 0.00387 http 0.00349 this 0.00341 project 0.00339 resource 0.00332 process 0.00325 support 0.00310 technology 0.00309 group 0.00307 may 0.00296 social 0.00272 TOPIC 3 word 0.02368 language 0.00790 type 0.00477 category 0.00407 corpus 0.00400 lexical 0.00371 this 0.00368 speech 0.00368 rule 0.00345 learner 0.00327 information 0.00318 form 0.00306 frequency 0.00301 lexicon 0.00297 analysis 0.00290 TOPIC 4 state 0.01214 process 0.01043 component 0.00740 rate 0.00677 pepa 0.00666 simulation 0.00662 stochastic 0.00581 agent 0.00488 modelling 0.00478 this 0.00451 analysis 0.00449 action 0.00433 behaviour 0.00419 transition 0.00390 reaction 0.00370 TOPIC 5 sentence 0.00874 word 0.00867 language 0.00797 text 0.00719 et 0.00659 corpus 0.00659 we 0.00654 al 0.00636 et_al 0.00505 feature 0.00470 verb 0.00400 np 0.00385 semantic 0.00361 example 0.00359 grammar 0.00353 TOPIC 6 agent 0.01292 action 0.00852 state 0.00520 game 0.00506 planning 0.00479 plan 0.00478 dialogue 0.00461 world 0.00400 problem 0.00399 knowledge 0.00389 domain 0.00379 this 0.00369 human 0.00351 example 0.00318 would 0.00310 TOPIC 7 state 0.00972 we 0.00962 probability 0.00897 lemma 0.00669 theorem 0.00662 let 0.00643 quantum 0.00601 function 0.00590 algorithm 0.00581 proof 0.00563 problem 0.00532 for 0.00481 polynomial 0.00467 given 0.00461 strategy 0.00452 TOPIC 8 gene 0.01178 protein 0.01062 cell 0.00962 al 0.00920 et 0.00889 et_al 0.00832 expression 0.00627 fig 0.00523 analysis 0.00414 mouse 0.00371 figure 0.00328 doi 0.00328 interaction 0.00300 biology 0.00287 level 0.00284 TOPIC 9 query 0.02685 we 0.01055 database 0.00877 algorithm 0.00842 problem 0.00744 graph 0.00640 attribute 0.00625 tuple 0.00575 pattern 0.00560 constraint 0.00554 view 0.00543 tuples 0.00537 instance 0.00524 given 0.00515 match 0.00508 TOPIC 10 participant 0.00720 word 0.00559 learning 0.00559 task 0.00516 effect 0.00508 experiment 0.00412 study 0.00399 child 0.00370 object 0.00349 this 0.00338 cognitive 0.00328 figure 0.00298 information 0.00283 language 0.00283 trial 0.00269 TOPIC 11 translation 0.01851 language 0.00854 de 0.00803 machine 0.00627 explorer 0.00539 word 0.00538 english 0.00508 computational 0.00503 association 0.00487 linguistics 0.00474 en 0.00459 phrase 0.00418 publication 0.00408 copyright 0.00404 machine_translation 0.00381 TOPIC 12 distribution 0.01102 algorithm 0.01079 we 0.00823 method 0.00762 value 0.00749 number 0.00712 parameter 0.00638 sample 0.00618 probability 0.00607 point 0.00535 figure 0.00499 cluster 0.00454 distance 0.00434 function 0.00413 space 0.00393 TOPIC 13 protocol 0.01562 security 0.00801 message 0.00729 we 0.00714 transition 0.00642 attack 0.00600 state 0.00566 key 0.00565 rule 0.00547 token 0.00407 cost 0.00379 program 0.00373 number 0.00366 case 0.00359 computation 0.00354 TOPIC 14 we 0.00776 algebra 0.00736 category 0.00726 let 0.00696 space 0.00554 structure 0.00552 theory 0.00547 theorem 0.00511 map 0.00482 element 0.00463 relation 0.00434 given 0.00434 proposition 0.00423 this 0.00417 object 0.00402 TOPIC 15 speech 0.03478 speaker 0.01287 voice 0.01137 synthesis 0.01087 speech_synthesis 0.00646 hmm 0.00576 pp 0.00471 synthetic 0.00470 sentence 0.00419 listener 0.00402 signal 0.00384 noise 0.00382 hmm_based 0.00374 test 0.00341 method 0.00338 TOPIC 16 node 0.02090 tree 0.01648 we 0.00913 graph 0.00880 query 0.00735 path 0.00727 xml 0.00711 state 0.00670 automaton 0.00564 edge 0.00542 for 0.00505 let 0.00496 problem 0.00468 language 0.00451 variable 0.00438 TOPIC 17 learning 0.01008 function 0.00636 parameter 0.00614 method 0.00603 network 0.00512 we 0.00510 training 0.00497 variable 0.00461 control 0.00421 motion 0.00420 distribution 0.00403 neural 0.00381 gaussian 0.00374 algorithm 0.00372 problem 0.00357 TOPIC 18 proof 0.02135 rule 0.00947 term 0.00853 theorem 0.00844 logic 0.00719 theory 0.00701 example 0.00511 we 0.00486 case 0.00485 this 0.00444 sp 0.00436 goal 0.00416 function 0.00413 order 0.00402 method 0.00362 TOPIC 19 object 0.02165 visual 0.01702 scene 0.01200 eye 0.00741 target 0.00608 processing 0.00562 task 0.00555 category 0.00525 movement 0.00516 information 0.00505 we 0.00501 feature 0.00451 context 0.00448 attention 0.00423 pattern 0.00420 TOPIC 20 image 0.02263 object 0.01232 class 0.00892 feature 0.00680 method 0.00666 we 0.00623 training 0.00530 detection 0.00508 video 0.00476 part 0.00425 this 0.00388 segmentation 0.00364 learning 0.00360 recognition 0.00352 dataset 0.00345 TOPIC 21 feature 0.01738 speech 0.01382 training 0.00746 acoustic 0.00732 recognition 0.00685 word 0.00662 network 0.00595 pp 0.00487 language 0.00478 speaker 0.00429 we 0.00409 neural 0.00404 meeting 0.00403 proc 0.00396 trained 0.00350 TOPIC 22 ontology 0.02185 transformation 0.01075 mapping 0.00598 concept 0.00589 example 0.00535 semantic 0.00533 language 0.00513 repair 0.00494 rdf 0.00438 relation 0.00424 source 0.00397 we 0.00395 change 0.00395 may 0.00389 knowledge 0.00388 TOPIC 23 cid 0.70005 signature 0.00510 de 0.00445 we 0.00337 let 0.00325 lemma 0.00299 case 0.00269 pl 0.00223 di 0.00175 then 0.00172 by 0.00170 arity 0.00163 type 0.00149 rst 0.00145 proof 0.00143 TOPIC 24 neuron 0.00896 stimulus 0.00643 response 0.00567 cell 0.00525 al 0.00521 et 0.00518 network 0.00487 activity 0.00484 et_al 0.00454 spike 0.00450 neural 0.00439 we 0.00410 input 0.00380 visual 0.00355 information 0.00355
Visualisation¶
In [36]:
vis_data = gensimvis.prepare(tmfull_best_model, corpus_tmfull, dict_tmfull)
/Users/weiting/miniconda3/envs/infnet3/lib/python3.6/site-packages/pyLDAvis/_prepare.py:387: DeprecationWarning: .ix is deprecated. Please use .loc for label based indexing or .iloc for positional indexing See the documentation here: http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated topic_term_dists = topic_term_dists.ix[topic_order]
In [37]:
pyLDAvis.display(vis_data)
Out[37]:
In [38]:
# save in html for viewing
pyLDAvis.save_html(vis_data, open('../src/res/tmfull/ldavis.html', 'w'))
tmfull_meta (tm-20yr) : Metadata only¶
In [8]:
dict_tmfull_meta = Dictionary.load(
os.path.join(DATA_DIR, 'corpora', 'dictionary.meta'))
df_metadata = df_combined_toks[['year', 'toks_metada']]
# Create a bow tagging for each publication:
df_metadata['bow'] = df_metadata['toks_metada'].apply(dict_tmfull_meta.doc2bow)
# Generate a corpus based on the tokens, which we will be using later
corpus_tmfull_meta = df_metadata.bow.tolist()
text_tmfull_meta = df_metadata.toks_metada.tolist()
2018-03-28 18:44:30,181 : INFO : loading Dictionary object from ../../data/data_schoolofinf/corpora/dictionary.meta 2018-03-28 18:44:30,282 : INFO : loaded ../../data/data_schoolofinf/corpora/dictionary.meta
In [46]:
tmfull_meta_score = pd.read_csv('../src/tmfull_meta/scores.csv')
Evaluation of model¶
In [47]:
tmfull_meta_score['c_v'] = tmfull_meta_score.apply(lambda row: get_score(row.num_topic, dict_tmfull_meta, texts=text_tmfull_meta, corpus=corpus_tmfull_meta,
coherence='c_v', model_dir='../src/tmfull_meta'),
axis=1)
2018-03-28 00:34:08,453 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_10 2018-03-28 00:34:08,533 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-28 00:34:08,554 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:34:08,555 : INFO : setting ignored attribute state to None 2018-03-28 00:34:08,555 : INFO : setting ignored attribute id2word to None 2018-03-28 00:34:08,556 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_10 2018-03-28 00:34:08,556 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_10.state 2018-03-28 00:34:08,581 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_10.state 2018-03-28 00:34:08,596 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:34:11,876 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:11,881 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:11,856 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:11,933 : INFO : accumulator serialized 2018-03-28 00:34:11,934 : INFO : accumulator serialized 2018-03-28 00:34:11,938 : INFO : accumulator serialized 2018-03-28 00:34:12,006 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:34:12,048 : INFO : accumulated word occurrence stats for 91588 virtual documents 2018-03-28 00:34:12,370 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_15 2018-03-28 00:34:12,384 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_15.expElogbeta.npy with mmap=None 2018-03-28 00:34:12,397 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:34:12,397 : INFO : setting ignored attribute state to None 2018-03-28 00:34:12,398 : INFO : setting ignored attribute id2word to None 2018-03-28 00:34:12,398 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_15 2018-03-28 00:34:12,399 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_15.state 2018-03-28 00:34:12,432 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_15.state 2018-03-28 00:34:12,455 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:34:17,459 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:17,475 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:17,480 : INFO : accumulator serialized 2018-03-28 00:34:17,466 : INFO : accumulator serialized 2018-03-28 00:34:17,544 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:17,550 : INFO : accumulator serialized 2018-03-28 00:34:17,609 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:34:17,684 : INFO : accumulated word occurrence stats for 91629 virtual documents 2018-03-28 00:34:18,164 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_20 2018-03-28 00:34:18,182 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_20.expElogbeta.npy with mmap=None 2018-03-28 00:34:18,191 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:34:18,192 : INFO : setting ignored attribute state to None 2018-03-28 00:34:18,192 : INFO : setting ignored attribute id2word to None 2018-03-28 00:34:18,193 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_20 2018-03-28 00:34:18,193 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_20.state 2018-03-28 00:34:18,224 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_20.state 2018-03-28 00:34:18,237 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:34:24,289 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:24,296 : INFO : accumulator serialized 2018-03-28 00:34:24,302 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:24,341 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:24,310 : INFO : accumulator serialized 2018-03-28 00:34:24,345 : INFO : accumulator serialized 2018-03-28 00:34:24,440 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:34:24,546 : INFO : accumulated word occurrence stats for 91639 virtual documents 2018-03-28 00:34:25,177 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_25 2018-03-28 00:34:25,216 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-28 00:34:25,227 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:34:25,227 : INFO : setting ignored attribute state to None 2018-03-28 00:34:25,228 : INFO : setting ignored attribute id2word to None 2018-03-28 00:34:25,228 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_25 2018-03-28 00:34:25,229 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_25.state 2018-03-28 00:34:25,275 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_25.state 2018-03-28 00:34:25,298 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:34:33,361 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:33,422 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:33,454 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:33,368 : INFO : accumulator serialized 2018-03-28 00:34:33,427 : INFO : accumulator serialized 2018-03-28 00:34:33,460 : INFO : accumulator serialized 2018-03-28 00:34:33,600 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:34:33,765 : INFO : accumulated word occurrence stats for 91652 virtual documents 2018-03-28 00:34:34,571 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_30 2018-03-28 00:34:34,609 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-28 00:34:34,622 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:34:34,622 : INFO : setting ignored attribute state to None 2018-03-28 00:34:34,623 : INFO : setting ignored attribute id2word to None 2018-03-28 00:34:34,624 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_30 2018-03-28 00:34:34,625 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_30.state 2018-03-28 00:34:34,668 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_30.state 2018-03-28 00:34:34,686 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:34:43,503 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:43,507 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:43,559 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:43,510 : INFO : accumulator serialized 2018-03-28 00:34:43,513 : INFO : accumulator serialized 2018-03-28 00:34:43,569 : INFO : accumulator serialized 2018-03-28 00:34:43,761 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:34:43,983 : INFO : accumulated word occurrence stats for 91725 virtual documents 2018-03-28 00:34:44,975 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_35 2018-03-28 00:34:45,010 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_35.expElogbeta.npy with mmap=None 2018-03-28 00:34:45,025 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:34:45,026 : INFO : setting ignored attribute state to None 2018-03-28 00:34:45,026 : INFO : setting ignored attribute id2word to None 2018-03-28 00:34:45,026 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_35 2018-03-28 00:34:45,027 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_35.state 2018-03-28 00:34:45,072 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_35.state 2018-03-28 00:34:45,088 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:34:55,083 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:55,121 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:55,089 : INFO : accumulator serialized 2018-03-28 00:34:55,127 : INFO : accumulator serialized 2018-03-28 00:34:55,300 : INFO : serializing accumulator to return to master... 2018-03-28 00:34:55,306 : INFO : accumulator serialized 2018-03-28 00:34:55,465 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:34:55,730 : INFO : accumulated word occurrence stats for 91731 virtual documents 2018-03-28 00:34:56,892 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_40 2018-03-28 00:34:56,937 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-28 00:34:56,953 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:34:56,954 : INFO : setting ignored attribute state to None 2018-03-28 00:34:56,954 : INFO : setting ignored attribute id2word to None 2018-03-28 00:34:56,954 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_40 2018-03-28 00:34:56,955 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_40.state 2018-03-28 00:34:57,012 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_40.state 2018-03-28 00:34:57,031 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:35:08,554 : INFO : serializing accumulator to return to master... 2018-03-28 00:35:08,731 : INFO : serializing accumulator to return to master... 2018-03-28 00:35:08,562 : INFO : accumulator serialized 2018-03-28 00:35:08,808 : INFO : serializing accumulator to return to master... 2018-03-28 00:35:08,739 : INFO : accumulator serialized 2018-03-28 00:35:08,813 : INFO : accumulator serialized 2018-03-28 00:35:09,028 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:35:09,377 : INFO : accumulated word occurrence stats for 91739 virtual documents 2018-03-28 00:35:10,724 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_50 2018-03-28 00:35:10,768 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_50.expElogbeta.npy with mmap=None 2018-03-28 00:35:10,790 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:35:10,791 : INFO : setting ignored attribute state to None 2018-03-28 00:35:10,791 : INFO : setting ignored attribute id2word to None 2018-03-28 00:35:10,792 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_50 2018-03-28 00:35:10,792 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_50.state 2018-03-28 00:35:10,867 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_50.state 2018-03-28 00:35:10,881 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:35:24,342 : INFO : serializing accumulator to return to master... 2018-03-28 00:35:24,360 : INFO : serializing accumulator to return to master... 2018-03-28 00:35:24,605 : INFO : serializing accumulator to return to master... 2018-03-28 00:35:24,347 : INFO : accumulator serialized 2018-03-28 00:35:24,367 : INFO : accumulator serialized 2018-03-28 00:35:24,611 : INFO : accumulator serialized 2018-03-28 00:35:24,925 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:35:25,412 : INFO : accumulated word occurrence stats for 91742 virtual documents
In [48]:
tmfull_meta_score['u_mass'] = tmfull_meta_score.apply(lambda row: get_score(row.num_topic, dict_tmfull_meta, texts=text_tmfull_meta, corpus=corpus_tmfull_meta,
coherence='u_mass', model_dir='../src/tmfull_meta'),
axis=1)
2018-03-28 00:35:28,365 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_10 2018-03-28 00:35:28,367 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-28 00:35:28,369 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:35:28,370 : INFO : setting ignored attribute state to None 2018-03-28 00:35:28,370 : INFO : setting ignored attribute id2word to None 2018-03-28 00:35:28,371 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_10 2018-03-28 00:35:28,371 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_10.state 2018-03-28 00:35:28,377 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_10.state 2018-03-28 00:35:28,403 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-28 00:35:28,423 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-28 00:35:28,443 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-28 00:35:28,464 : INFO : CorpusAccumulator accumulated stats from 4000 documents 2018-03-28 00:35:28,487 : INFO : CorpusAccumulator accumulated stats from 5000 documents 2018-03-28 00:35:28,511 : INFO : CorpusAccumulator accumulated stats from 6000 documents 2018-03-28 00:35:28,533 : INFO : CorpusAccumulator accumulated stats from 7000 documents 2018-03-28 00:35:28,557 : INFO : CorpusAccumulator accumulated stats from 8000 documents 2018-03-28 00:35:28,602 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_15 2018-03-28 00:35:28,603 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_15.expElogbeta.npy with mmap=None 2018-03-28 00:35:28,606 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:35:28,606 : INFO : setting ignored attribute state to None 2018-03-28 00:35:28,607 : INFO : setting ignored attribute id2word to None 2018-03-28 00:35:28,607 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_15 2018-03-28 00:35:28,608 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_15.state 2018-03-28 00:35:28,616 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_15.state 2018-03-28 00:35:28,645 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-28 00:35:28,667 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-28 00:35:28,689 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-28 00:35:28,710 : INFO : CorpusAccumulator accumulated stats from 4000 documents 2018-03-28 00:35:28,732 : INFO : CorpusAccumulator accumulated stats from 5000 documents 2018-03-28 00:35:28,755 : INFO : CorpusAccumulator accumulated stats from 6000 documents 2018-03-28 00:35:28,778 : INFO : CorpusAccumulator accumulated stats from 7000 documents 2018-03-28 00:35:28,802 : INFO : CorpusAccumulator accumulated stats from 8000 documents 2018-03-28 00:35:28,857 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_20 2018-03-28 00:35:28,858 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_20.expElogbeta.npy with mmap=None 2018-03-28 00:35:28,860 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:35:28,860 : INFO : setting ignored attribute state to None 2018-03-28 00:35:28,861 : INFO : setting ignored attribute id2word to None 2018-03-28 00:35:28,862 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_20 2018-03-28 00:35:28,862 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_20.state 2018-03-28 00:35:28,872 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_20.state 2018-03-28 00:35:28,902 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-28 00:35:28,924 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-28 00:35:28,947 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-28 00:35:28,969 : INFO : CorpusAccumulator accumulated stats from 4000 documents 2018-03-28 00:35:28,992 : INFO : CorpusAccumulator accumulated stats from 5000 documents 2018-03-28 00:35:29,016 : INFO : CorpusAccumulator accumulated stats from 6000 documents 2018-03-28 00:35:29,040 : INFO : CorpusAccumulator accumulated stats from 7000 documents 2018-03-28 00:35:29,064 : INFO : CorpusAccumulator accumulated stats from 8000 documents 2018-03-28 00:35:29,132 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_25 2018-03-28 00:35:29,133 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-28 00:35:29,135 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:35:29,136 : INFO : setting ignored attribute state to None 2018-03-28 00:35:29,136 : INFO : setting ignored attribute id2word to None 2018-03-28 00:35:29,137 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_25 2018-03-28 00:35:29,137 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_25.state 2018-03-28 00:35:29,148 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_25.state 2018-03-28 00:35:29,182 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-28 00:35:29,207 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-28 00:35:29,232 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-28 00:35:29,256 : INFO : CorpusAccumulator accumulated stats from 4000 documents 2018-03-28 00:35:29,281 : INFO : CorpusAccumulator accumulated stats from 5000 documents 2018-03-28 00:35:29,307 : INFO : CorpusAccumulator accumulated stats from 6000 documents 2018-03-28 00:35:29,333 : INFO : CorpusAccumulator accumulated stats from 7000 documents 2018-03-28 00:35:29,359 : INFO : CorpusAccumulator accumulated stats from 8000 documents 2018-03-28 00:35:29,437 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_30 2018-03-28 00:35:29,439 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-28 00:35:29,440 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:35:29,441 : INFO : setting ignored attribute state to None 2018-03-28 00:35:29,442 : INFO : setting ignored attribute id2word to None 2018-03-28 00:35:29,442 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_30 2018-03-28 00:35:29,443 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_30.state 2018-03-28 00:35:29,457 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_30.state 2018-03-28 00:35:29,492 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-28 00:35:29,517 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-28 00:35:29,542 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-28 00:35:29,567 : INFO : CorpusAccumulator accumulated stats from 4000 documents 2018-03-28 00:35:29,593 : INFO : CorpusAccumulator accumulated stats from 5000 documents 2018-03-28 00:35:29,620 : INFO : CorpusAccumulator accumulated stats from 6000 documents 2018-03-28 00:35:29,647 : INFO : CorpusAccumulator accumulated stats from 7000 documents 2018-03-28 00:35:29,674 : INFO : CorpusAccumulator accumulated stats from 8000 documents 2018-03-28 00:35:29,761 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_35 2018-03-28 00:35:29,763 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_35.expElogbeta.npy with mmap=None 2018-03-28 00:35:29,764 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:35:29,765 : INFO : setting ignored attribute state to None 2018-03-28 00:35:29,766 : INFO : setting ignored attribute id2word to None 2018-03-28 00:35:29,766 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_35 2018-03-28 00:35:29,767 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_35.state 2018-03-28 00:35:29,782 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_35.state 2018-03-28 00:35:29,819 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-28 00:35:29,846 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-28 00:35:29,872 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-28 00:35:29,897 : INFO : CorpusAccumulator accumulated stats from 4000 documents 2018-03-28 00:35:29,923 : INFO : CorpusAccumulator accumulated stats from 5000 documents 2018-03-28 00:35:29,951 : INFO : CorpusAccumulator accumulated stats from 6000 documents 2018-03-28 00:35:29,978 : INFO : CorpusAccumulator accumulated stats from 7000 documents 2018-03-28 00:35:30,006 : INFO : CorpusAccumulator accumulated stats from 8000 documents 2018-03-28 00:35:30,105 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_40 2018-03-28 00:35:30,106 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-28 00:35:30,108 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:35:30,108 : INFO : setting ignored attribute state to None 2018-03-28 00:35:30,109 : INFO : setting ignored attribute id2word to None 2018-03-28 00:35:30,109 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_40 2018-03-28 00:35:30,110 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_40.state 2018-03-28 00:35:30,129 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_40.state 2018-03-28 00:35:30,169 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-28 00:35:30,196 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-28 00:35:30,222 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-28 00:35:30,248 : INFO : CorpusAccumulator accumulated stats from 4000 documents 2018-03-28 00:35:30,275 : INFO : CorpusAccumulator accumulated stats from 5000 documents 2018-03-28 00:35:30,303 : INFO : CorpusAccumulator accumulated stats from 6000 documents 2018-03-28 00:35:30,330 : INFO : CorpusAccumulator accumulated stats from 7000 documents 2018-03-28 00:35:30,358 : INFO : CorpusAccumulator accumulated stats from 8000 documents 2018-03-28 00:35:30,467 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_50 2018-03-28 00:35:30,468 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_50.expElogbeta.npy with mmap=None 2018-03-28 00:35:30,470 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:35:30,471 : INFO : setting ignored attribute state to None 2018-03-28 00:35:30,471 : INFO : setting ignored attribute id2word to None 2018-03-28 00:35:30,472 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_50 2018-03-28 00:35:30,472 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_50.state 2018-03-28 00:35:30,495 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_50.state 2018-03-28 00:35:30,539 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-28 00:35:30,567 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-28 00:35:30,595 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-28 00:35:30,622 : INFO : CorpusAccumulator accumulated stats from 4000 documents 2018-03-28 00:35:30,650 : INFO : CorpusAccumulator accumulated stats from 5000 documents 2018-03-28 00:35:30,679 : INFO : CorpusAccumulator accumulated stats from 6000 documents 2018-03-28 00:35:30,708 : INFO : CorpusAccumulator accumulated stats from 7000 documents 2018-03-28 00:35:30,737 : INFO : CorpusAccumulator accumulated stats from 8000 documents
In [49]:
tmfull_meta_score['c_npmi'] = tmfull_meta_score.apply(lambda row: get_score(row.num_topic, dict_tmfull_meta, texts=text_tmfull_meta, corpus=corpus_tmfull_meta,
coherence='c_npmi', model_dir='../src/tmfull_meta'),
axis=1)
2018-03-28 00:35:30,863 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_10 2018-03-28 00:35:30,865 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-28 00:35:30,867 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:35:30,868 : INFO : setting ignored attribute state to None 2018-03-28 00:35:30,868 : INFO : setting ignored attribute id2word to None 2018-03-28 00:35:30,869 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_10 2018-03-28 00:35:30,869 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_10.state 2018-03-28 00:35:30,876 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_10.state 2018-03-28 00:35:30,884 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:35:30,939 : INFO : 1 batches submitted to accumulate stats from 64 documents (4982 virtual) 2018-03-28 00:35:30,946 : INFO : 2 batches submitted to accumulate stats from 128 documents (10157 virtual) 2018-03-28 00:35:30,952 : INFO : 3 batches submitted to accumulate stats from 192 documents (15057 virtual) 2018-03-28 00:35:30,959 : INFO : 4 batches submitted to accumulate stats from 256 documents (20243 virtual) 2018-03-28 00:35:30,966 : INFO : 5 batches submitted to accumulate stats from 320 documents (25109 virtual) 2018-03-28 00:35:30,973 : INFO : 6 batches submitted to accumulate stats from 384 documents (30729 virtual) 2018-03-28 00:35:31,088 : INFO : 7 batches submitted to accumulate stats from 448 documents (36045 virtual) 2018-03-28 00:35:31,100 : INFO : 8 batches submitted to accumulate stats from 512 documents (40896 virtual) 2018-03-28 00:35:31,115 : INFO : 9 batches submitted to accumulate stats from 576 documents (45761 virtual) 2018-03-28 00:35:31,222 : INFO : 10 batches submitted to accumulate stats from 640 documents (51266 virtual) 2018-03-28 00:35:31,244 : INFO : 11 batches submitted to accumulate stats from 704 documents (56807 virtual) 2018-03-28 00:35:31,278 : INFO : 12 batches submitted to accumulate stats from 768 documents (61339 virtual) 2018-03-28 00:35:31,384 : INFO : 13 batches submitted to accumulate stats from 832 documents (67243 virtual) 2018-03-28 00:35:31,393 : INFO : 14 batches submitted to accumulate stats from 896 documents (72935 virtual) 2018-03-28 00:35:31,399 : INFO : 15 batches submitted to accumulate stats from 960 documents (78230 virtual) 2018-03-28 00:35:31,515 : INFO : 16 batches submitted to accumulate stats from 1024 documents (83975 virtual) 2018-03-28 00:35:31,527 : INFO : 17 batches submitted to accumulate stats from 1088 documents (89092 virtual) 2018-03-28 00:35:31,562 : INFO : 18 batches submitted to accumulate stats from 1152 documents (94203 virtual) 2018-03-28 00:35:31,669 : INFO : 19 batches submitted to accumulate stats from 1216 documents (99097 virtual) 2018-03-28 00:35:31,683 : INFO : 20 batches submitted to accumulate stats from 1280 documents (104698 virtual) 2018-03-28 00:35:31,723 : INFO : 21 batches submitted to accumulate stats from 1344 documents (110426 virtual) 2018-03-28 00:35:31,815 : INFO : 22 batches submitted to accumulate stats from 1408 documents (115651 virtual) 2018-03-28 00:35:31,836 : INFO : 23 batches submitted to accumulate stats from 1472 documents (121606 virtual) 2018-03-28 00:35:31,854 : INFO : 24 batches submitted to accumulate stats from 1536 documents (126947 virtual) 2018-03-28 00:35:31,960 : INFO : 25 batches submitted to accumulate stats from 1600 documents (132136 virtual) 2018-03-28 00:35:31,984 : INFO : 26 batches submitted to accumulate stats from 1664 documents (136767 virtual) 2018-03-28 00:35:32,000 : INFO : 27 batches submitted to accumulate stats from 1728 documents (142089 virtual) 2018-03-28 00:35:32,117 : INFO : 28 batches submitted to accumulate stats from 1792 documents (147605 virtual) 2018-03-28 00:35:32,138 : INFO : 29 batches submitted to accumulate stats from 1856 documents (153377 virtual) 2018-03-28 00:35:32,148 : INFO : 30 batches submitted to accumulate stats from 1920 documents (158669 virtual) 2018-03-28 00:35:32,239 : INFO : 31 batches submitted to accumulate stats from 1984 documents (163835 virtual) 2018-03-28 00:35:32,265 : INFO : 32 batches submitted to accumulate stats from 2048 documents (168727 virtual) 2018-03-28 00:35:32,307 : INFO : 33 batches submitted to accumulate stats from 2112 documents (173687 virtual) 2018-03-28 00:35:32,396 : INFO : 34 batches submitted to accumulate stats from 2176 documents (179414 virtual) 2018-03-28 00:35:32,419 : INFO : 35 batches submitted to accumulate stats from 2240 documents (184912 virtual) 2018-03-28 00:35:32,456 : INFO : 36 batches submitted to accumulate stats from 2304 documents (190083 virtual) 2018-03-28 00:35:32,536 : INFO : 37 batches submitted to accumulate stats from 2368 documents (195896 virtual) 2018-03-28 00:35:32,550 : INFO : 38 batches submitted to accumulate stats from 2432 documents (201785 virtual) 2018-03-28 00:35:32,607 : INFO : 39 batches submitted to accumulate stats from 2496 documents (206828 virtual) 2018-03-28 00:35:32,688 : INFO : 40 batches submitted to accumulate stats from 2560 documents (212216 virtual) 2018-03-28 00:35:32,700 : INFO : 41 batches submitted to accumulate stats from 2624 documents (216767 virtual) 2018-03-28 00:35:32,754 : INFO : 42 batches submitted to accumulate stats from 2688 documents (222176 virtual) 2018-03-28 00:35:32,843 : INFO : 43 batches submitted to accumulate stats from 2752 documents (226566 virtual) 2018-03-28 00:35:32,849 : INFO : 44 batches submitted to accumulate stats from 2816 documents (231686 virtual) 2018-03-28 00:35:32,898 : INFO : 45 batches submitted to accumulate stats from 2880 documents (236732 virtual) 2018-03-28 00:35:32,967 : INFO : 46 batches submitted to accumulate stats from 2944 documents (241514 virtual) 2018-03-28 00:35:32,995 : INFO : 47 batches submitted to accumulate stats from 3008 documents (246552 virtual) 2018-03-28 00:35:33,039 : INFO : 48 batches submitted to accumulate stats from 3072 documents (251553 virtual) 2018-03-28 00:35:33,097 : INFO : 49 batches submitted to accumulate stats from 3136 documents (256422 virtual) 2018-03-28 00:35:33,158 : INFO : 50 batches submitted to accumulate stats from 3200 documents (262545 virtual) 2018-03-28 00:35:33,165 : INFO : 51 batches submitted to accumulate stats from 3264 documents (267244 virtual) 2018-03-28 00:35:33,233 : INFO : 52 batches submitted to accumulate stats from 3328 documents (272652 virtual) 2018-03-28 00:35:33,293 : INFO : 53 batches submitted to accumulate stats from 3392 documents (277830 virtual) 2018-03-28 00:35:33,310 : INFO : 54 batches submitted to accumulate stats from 3456 documents (282897 virtual) 2018-03-28 00:35:33,346 : INFO : 55 batches submitted to accumulate stats from 3520 documents (288096 virtual) 2018-03-28 00:35:33,434 : INFO : 56 batches submitted to accumulate stats from 3584 documents (292972 virtual) 2018-03-28 00:35:33,472 : INFO : 57 batches submitted to accumulate stats from 3648 documents (298753 virtual) 2018-03-28 00:35:33,499 : INFO : 58 batches submitted to accumulate stats from 3712 documents (303929 virtual) 2018-03-28 00:35:33,568 : INFO : 59 batches submitted to accumulate stats from 3776 documents (308877 virtual) 2018-03-28 00:35:33,608 : INFO : 60 batches submitted to accumulate stats from 3840 documents (314413 virtual) 2018-03-28 00:35:33,653 : INFO : 61 batches submitted to accumulate stats from 3904 documents (319313 virtual) 2018-03-28 00:35:33,692 : INFO : 62 batches submitted to accumulate stats from 3968 documents (323950 virtual) 2018-03-28 00:35:33,762 : INFO : 63 batches submitted to accumulate stats from 4032 documents (329116 virtual) 2018-03-28 00:35:33,808 : INFO : 64 batches submitted to accumulate stats from 4096 documents (334615 virtual) 2018-03-28 00:35:33,823 : INFO : 65 batches submitted to accumulate stats from 4160 documents (340062 virtual) 2018-03-28 00:35:33,907 : INFO : 66 batches submitted to accumulate stats from 4224 documents (345064 virtual) 2018-03-28 00:35:33,951 : INFO : 67 batches submitted to accumulate stats from 4288 documents (350313 virtual) 2018-03-28 00:35:33,957 : INFO : 68 batches submitted to accumulate stats from 4352 documents (355588 virtual) 2018-03-28 00:35:34,078 : INFO : 69 batches submitted to accumulate stats from 4416 documents (360644 virtual) 2018-03-28 00:35:34,138 : INFO : 70 batches submitted to accumulate stats from 4480 documents (365848 virtual) 2018-03-28 00:35:34,144 : INFO : 71 batches submitted to accumulate stats from 4544 documents (370526 virtual) 2018-03-28 00:35:34,223 : INFO : 72 batches submitted to accumulate stats from 4608 documents (375897 virtual) 2018-03-28 00:35:34,285 : INFO : 73 batches submitted to accumulate stats from 4672 documents (381628 virtual) 2018-03-28 00:35:34,305 : INFO : 74 batches submitted to accumulate stats from 4736 documents (386903 virtual) 2018-03-28 00:35:34,352 : INFO : 75 batches submitted to accumulate stats from 4800 documents (391772 virtual) 2018-03-28 00:35:34,416 : INFO : 76 batches submitted to accumulate stats from 4864 documents (396642 virtual) 2018-03-28 00:35:34,449 : INFO : 77 batches submitted to accumulate stats from 4928 documents (402139 virtual) 2018-03-28 00:35:34,490 : INFO : 78 batches submitted to accumulate stats from 4992 documents (406987 virtual) 2018-03-28 00:35:34,569 : INFO : 79 batches submitted to accumulate stats from 5056 documents (412433 virtual) 2018-03-28 00:35:34,594 : INFO : 80 batches submitted to accumulate stats from 5120 documents (417859 virtual) 2018-03-28 00:35:34,647 : INFO : 81 batches submitted to accumulate stats from 5184 documents (423375 virtual) 2018-03-28 00:35:34,691 : INFO : 82 batches submitted to accumulate stats from 5248 documents (428518 virtual) 2018-03-28 00:35:34,743 : INFO : 83 batches submitted to accumulate stats from 5312 documents (434380 virtual) 2018-03-28 00:35:34,779 : INFO : 84 batches submitted to accumulate stats from 5376 documents (440012 virtual) 2018-03-28 00:35:34,845 : INFO : 85 batches submitted to accumulate stats from 5440 documents (445882 virtual) 2018-03-28 00:35:34,884 : INFO : 86 batches submitted to accumulate stats from 5504 documents (450244 virtual) 2018-03-28 00:35:34,950 : INFO : 87 batches submitted to accumulate stats from 5568 documents (455808 virtual) 2018-03-28 00:35:34,968 : INFO : 88 batches submitted to accumulate stats from 5632 documents (460886 virtual) 2018-03-28 00:35:35,032 : INFO : 89 batches submitted to accumulate stats from 5696 documents (466358 virtual) 2018-03-28 00:35:35,116 : INFO : 90 batches submitted to accumulate stats from 5760 documents (471842 virtual) 2018-03-28 00:35:35,131 : INFO : 91 batches submitted to accumulate stats from 5824 documents (477442 virtual) 2018-03-28 00:35:35,163 : INFO : 92 batches submitted to accumulate stats from 5888 documents (482773 virtual) 2018-03-28 00:35:35,252 : INFO : 93 batches submitted to accumulate stats from 5952 documents (488229 virtual) 2018-03-28 00:35:35,279 : INFO : 94 batches submitted to accumulate stats from 6016 documents (493653 virtual) 2018-03-28 00:35:35,312 : INFO : 95 batches submitted to accumulate stats from 6080 documents (499329 virtual) 2018-03-28 00:35:35,413 : INFO : 96 batches submitted to accumulate stats from 6144 documents (504582 virtual) 2018-03-28 00:35:35,436 : INFO : 97 batches submitted to accumulate stats from 6208 documents (509569 virtual) 2018-03-28 00:35:35,458 : INFO : 98 batches submitted to accumulate stats from 6272 documents (514533 virtual) 2018-03-28 00:35:35,551 : INFO : 99 batches submitted to accumulate stats from 6336 documents (519517 virtual) 2018-03-28 00:35:35,580 : INFO : 100 batches submitted to accumulate stats from 6400 documents (524779 virtual) 2018-03-28 00:35:35,627 : INFO : 101 batches submitted to accumulate stats from 6464 documents (529613 virtual) 2018-03-28 00:35:35,668 : INFO : 102 batches submitted to accumulate stats from 6528 documents (534611 virtual) 2018-03-28 00:35:35,738 : INFO : 103 batches submitted to accumulate stats from 6592 documents (540049 virtual) 2018-03-28 00:35:35,791 : INFO : 104 batches submitted to accumulate stats from 6656 documents (545389 virtual) 2018-03-28 00:35:35,802 : INFO : 105 batches submitted to accumulate stats from 6720 documents (550182 virtual) 2018-03-28 00:35:35,858 : INFO : 106 batches submitted to accumulate stats from 6784 documents (555700 virtual) 2018-03-28 00:35:35,927 : INFO : 107 batches submitted to accumulate stats from 6848 documents (560813 virtual) 2018-03-28 00:35:35,957 : INFO : 108 batches submitted to accumulate stats from 6912 documents (566484 virtual) 2018-03-28 00:35:35,998 : INFO : 109 batches submitted to accumulate stats from 6976 documents (571428 virtual) 2018-03-28 00:35:36,075 : INFO : 110 batches submitted to accumulate stats from 7040 documents (576165 virtual) 2018-03-28 00:35:36,083 : INFO : 111 batches submitted to accumulate stats from 7104 documents (580954 virtual) 2018-03-28 00:35:36,176 : INFO : 112 batches submitted to accumulate stats from 7168 documents (586430 virtual) 2018-03-28 00:35:36,217 : INFO : 113 batches submitted to accumulate stats from 7232 documents (591062 virtual) 2018-03-28 00:35:36,225 : INFO : 114 batches submitted to accumulate stats from 7296 documents (596601 virtual) 2018-03-28 00:35:36,321 : INFO : 115 batches submitted to accumulate stats from 7360 documents (602146 virtual) 2018-03-28 00:35:36,343 : INFO : 116 batches submitted to accumulate stats from 7424 documents (607081 virtual) 2018-03-28 00:35:36,357 : INFO : 117 batches submitted to accumulate stats from 7488 documents (612148 virtual) 2018-03-28 00:35:36,456 : INFO : 118 batches submitted to accumulate stats from 7552 documents (617413 virtual) 2018-03-28 00:35:36,479 : INFO : 119 batches submitted to accumulate stats from 7616 documents (622732 virtual) 2018-03-28 00:35:36,545 : INFO : 120 batches submitted to accumulate stats from 7680 documents (627979 virtual) 2018-03-28 00:35:36,598 : INFO : 121 batches submitted to accumulate stats from 7744 documents (633848 virtual) 2018-03-28 00:35:36,618 : INFO : 122 batches submitted to accumulate stats from 7808 documents (638824 virtual) 2018-03-28 00:35:36,695 : INFO : 123 batches submitted to accumulate stats from 7872 documents (643881 virtual) 2018-03-28 00:35:36,747 : INFO : 124 batches submitted to accumulate stats from 7936 documents (647126 virtual) 2018-03-28 00:35:36,928 : INFO : serializing accumulator to return to master... 2018-03-28 00:35:36,952 : INFO : serializing accumulator to return to master... 2018-03-28 00:35:36,934 : INFO : accumulator serialized 2018-03-28 00:35:36,967 : INFO : serializing accumulator to return to master... 2018-03-28 00:35:36,958 : INFO : accumulator serialized 2018-03-28 00:35:36,974 : INFO : accumulator serialized 2018-03-28 00:35:37,018 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:35:37,054 : INFO : accumulated word occurrence stats for 647659 virtual documents 2018-03-28 00:35:37,358 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_15 2018-03-28 00:35:37,360 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_15.expElogbeta.npy with mmap=None 2018-03-28 00:35:37,363 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:35:37,364 : INFO : setting ignored attribute state to None 2018-03-28 00:35:37,364 : INFO : setting ignored attribute id2word to None 2018-03-28 00:35:37,365 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_15 2018-03-28 00:35:37,365 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_15.state 2018-03-28 00:35:37,374 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_15.state 2018-03-28 00:35:37,385 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:35:37,430 : INFO : 1 batches submitted to accumulate stats from 64 documents (4982 virtual) 2018-03-28 00:35:37,438 : INFO : 2 batches submitted to accumulate stats from 128 documents (10157 virtual) 2018-03-28 00:35:37,445 : INFO : 3 batches submitted to accumulate stats from 192 documents (15057 virtual) 2018-03-28 00:35:37,452 : INFO : 4 batches submitted to accumulate stats from 256 documents (19941 virtual) 2018-03-28 00:35:37,460 : INFO : 5 batches submitted to accumulate stats from 320 documents (25048 virtual) 2018-03-28 00:35:37,467 : INFO : 6 batches submitted to accumulate stats from 384 documents (30581 virtual) 2018-03-28 00:35:37,609 : INFO : 7 batches submitted to accumulate stats from 448 documents (35845 virtual) 2018-03-28 00:35:37,615 : INFO : 8 batches submitted to accumulate stats from 512 documents (40777 virtual) 2018-03-28 00:35:37,624 : INFO : 9 batches submitted to accumulate stats from 576 documents (45638 virtual) 2018-03-28 00:35:37,739 : INFO : 10 batches submitted to accumulate stats from 640 documents (51118 virtual) 2018-03-28 00:35:37,782 : INFO : 11 batches submitted to accumulate stats from 704 documents (56409 virtual) 2018-03-28 00:35:37,804 : INFO : 12 batches submitted to accumulate stats from 768 documents (61058 virtual) 2018-03-28 00:35:37,917 : INFO : 13 batches submitted to accumulate stats from 832 documents (66886 virtual) 2018-03-28 00:35:37,924 : INFO : 14 batches submitted to accumulate stats from 896 documents (72586 virtual) 2018-03-28 00:35:37,977 : INFO : 15 batches submitted to accumulate stats from 960 documents (77877 virtual) 2018-03-28 00:35:38,066 : INFO : 16 batches submitted to accumulate stats from 1024 documents (83673 virtual) 2018-03-28 00:35:38,099 : INFO : 17 batches submitted to accumulate stats from 1088 documents (88804 virtual) 2018-03-28 00:35:38,151 : INFO : 18 batches submitted to accumulate stats from 1152 documents (93954 virtual) 2018-03-28 00:35:38,210 : INFO : 19 batches submitted to accumulate stats from 1216 documents (98617 virtual) 2018-03-28 00:35:38,281 : INFO : 20 batches submitted to accumulate stats from 1280 documents (104250 virtual) 2018-03-28 00:35:38,325 : INFO : 21 batches submitted to accumulate stats from 1344 documents (109784 virtual) 2018-03-28 00:35:38,391 : INFO : 22 batches submitted to accumulate stats from 1408 documents (114780 virtual) 2018-03-28 00:35:38,453 : INFO : 23 batches submitted to accumulate stats from 1472 documents (120626 virtual) 2018-03-28 00:35:38,478 : INFO : 24 batches submitted to accumulate stats from 1536 documents (126158 virtual) 2018-03-28 00:35:38,523 : INFO : 25 batches submitted to accumulate stats from 1600 documents (131438 virtual) 2018-03-28 00:35:38,630 : INFO : 26 batches submitted to accumulate stats from 1664 documents (135958 virtual) 2018-03-28 00:35:38,651 : INFO : 27 batches submitted to accumulate stats from 1728 documents (141622 virtual) 2018-03-28 00:35:38,700 : INFO : 28 batches submitted to accumulate stats from 1792 documents (146695 virtual) 2018-03-28 00:35:38,789 : INFO : 29 batches submitted to accumulate stats from 1856 documents (152488 virtual) 2018-03-28 00:35:38,826 : INFO : 30 batches submitted to accumulate stats from 1920 documents (158014 virtual) 2018-03-28 00:35:38,873 : INFO : 31 batches submitted to accumulate stats from 1984 documents (163034 virtual) 2018-03-28 00:35:38,942 : INFO : 32 batches submitted to accumulate stats from 2048 documents (168065 virtual) 2018-03-28 00:35:38,989 : INFO : 33 batches submitted to accumulate stats from 2112 documents (172956 virtual) 2018-03-28 00:35:39,031 : INFO : 34 batches submitted to accumulate stats from 2176 documents (178439 virtual) 2018-03-28 00:35:39,111 : INFO : 35 batches submitted to accumulate stats from 2240 documents (183952 virtual) 2018-03-28 00:35:39,154 : INFO : 36 batches submitted to accumulate stats from 2304 documents (189020 virtual) 2018-03-28 00:35:39,199 : INFO : 37 batches submitted to accumulate stats from 2368 documents (195016 virtual) 2018-03-28 00:35:39,275 : INFO : 38 batches submitted to accumulate stats from 2432 documents (200909 virtual) 2018-03-28 00:35:39,314 : INFO : 39 batches submitted to accumulate stats from 2496 documents (205941 virtual) 2018-03-28 00:35:39,376 : INFO : 40 batches submitted to accumulate stats from 2560 documents (211023 virtual) 2018-03-28 00:35:39,434 : INFO : 41 batches submitted to accumulate stats from 2624 documents (215691 virtual) 2018-03-28 00:35:39,460 : INFO : 42 batches submitted to accumulate stats from 2688 documents (221215 virtual) 2018-03-28 00:35:39,555 : INFO : 43 batches submitted to accumulate stats from 2752 documents (225418 virtual) 2018-03-28 00:35:39,610 : INFO : 44 batches submitted to accumulate stats from 2816 documents (230340 virtual) 2018-03-28 00:35:39,631 : INFO : 45 batches submitted to accumulate stats from 2880 documents (235829 virtual) 2018-03-28 00:35:39,707 : INFO : 46 batches submitted to accumulate stats from 2944 documents (240875 virtual) 2018-03-28 00:35:39,758 : INFO : 47 batches submitted to accumulate stats from 3008 documents (245402 virtual) 2018-03-28 00:35:39,801 : INFO : 48 batches submitted to accumulate stats from 3072 documents (250356 virtual) 2018-03-28 00:35:39,843 : INFO : 49 batches submitted to accumulate stats from 3136 documents (255295 virtual) 2018-03-28 00:35:39,909 : INFO : 50 batches submitted to accumulate stats from 3200 documents (261177 virtual) 2018-03-28 00:35:39,978 : INFO : 51 batches submitted to accumulate stats from 3264 documents (266149 virtual) 2018-03-28 00:35:40,018 : INFO : 52 batches submitted to accumulate stats from 3328 documents (271374 virtual) 2018-03-28 00:35:40,034 : INFO : 53 batches submitted to accumulate stats from 3392 documents (276331 virtual) 2018-03-28 00:35:40,140 : INFO : 54 batches submitted to accumulate stats from 3456 documents (281542 virtual) 2018-03-28 00:35:40,173 : INFO : 55 batches submitted to accumulate stats from 3520 documents (286462 virtual) 2018-03-28 00:35:40,203 : INFO : 56 batches submitted to accumulate stats from 3584 documents (291443 virtual) 2018-03-28 00:35:40,290 : INFO : 57 batches submitted to accumulate stats from 3648 documents (297042 virtual) 2018-03-28 00:35:40,342 : INFO : 58 batches submitted to accumulate stats from 3712 documents (302255 virtual) 2018-03-28 00:35:40,359 : INFO : 59 batches submitted to accumulate stats from 3776 documents (307481 virtual) 2018-03-28 00:35:40,434 : INFO : 60 batches submitted to accumulate stats from 3840 documents (312618 virtual) 2018-03-28 00:35:40,500 : INFO : 61 batches submitted to accumulate stats from 3904 documents (317936 virtual) 2018-03-28 00:35:40,523 : INFO : 62 batches submitted to accumulate stats from 3968 documents (322678 virtual) 2018-03-28 00:35:40,600 : INFO : 63 batches submitted to accumulate stats from 4032 documents (327278 virtual) 2018-03-28 00:35:40,667 : INFO : 64 batches submitted to accumulate stats from 4096 documents (332843 virtual) 2018-03-28 00:35:40,676 : INFO : 65 batches submitted to accumulate stats from 4160 documents (337816 virtual) 2018-03-28 00:35:40,772 : INFO : 66 batches submitted to accumulate stats from 4224 documents (343147 virtual) 2018-03-28 00:35:40,838 : INFO : 67 batches submitted to accumulate stats from 4288 documents (348325 virtual) 2018-03-28 00:35:40,844 : INFO : 68 batches submitted to accumulate stats from 4352 documents (353585 virtual) 2018-03-28 00:35:40,916 : INFO : 69 batches submitted to accumulate stats from 4416 documents (358769 virtual) 2018-03-28 00:35:40,989 : INFO : 70 batches submitted to accumulate stats from 4480 documents (363761 virtual) 2018-03-28 00:35:41,024 : INFO : 71 batches submitted to accumulate stats from 4544 documents (368526 virtual) 2018-03-28 00:35:41,093 : INFO : 72 batches submitted to accumulate stats from 4608 documents (373588 virtual) 2018-03-28 00:35:41,138 : INFO : 73 batches submitted to accumulate stats from 4672 documents (378928 virtual) 2018-03-28 00:35:41,184 : INFO : 74 batches submitted to accumulate stats from 4736 documents (384334 virtual) 2018-03-28 00:35:41,242 : INFO : 75 batches submitted to accumulate stats from 4800 documents (389837 virtual) 2018-03-28 00:35:41,310 : INFO : 76 batches submitted to accumulate stats from 4864 documents (394001 virtual) 2018-03-28 00:35:41,340 : INFO : 77 batches submitted to accumulate stats from 4928 documents (399863 virtual) 2018-03-28 00:35:41,401 : INFO : 78 batches submitted to accumulate stats from 4992 documents (404651 virtual) 2018-03-28 00:35:41,473 : INFO : 79 batches submitted to accumulate stats from 5056 documents (409835 virtual) 2018-03-28 00:35:41,508 : INFO : 80 batches submitted to accumulate stats from 5120 documents (415102 virtual) 2018-03-28 00:35:41,565 : INFO : 81 batches submitted to accumulate stats from 5184 documents (420818 virtual) 2018-03-28 00:35:41,612 : INFO : 82 batches submitted to accumulate stats from 5248 documents (426106 virtual) 2018-03-28 00:35:41,683 : INFO : 83 batches submitted to accumulate stats from 5312 documents (431416 virtual) 2018-03-28 00:35:41,718 : INFO : 84 batches submitted to accumulate stats from 5376 documents (437077 virtual) 2018-03-28 00:35:41,781 : INFO : 85 batches submitted to accumulate stats from 5440 documents (442566 virtual) 2018-03-28 00:35:41,836 : INFO : 86 batches submitted to accumulate stats from 5504 documents (448354 virtual) 2018-03-28 00:35:41,895 : INFO : 87 batches submitted to accumulate stats from 5568 documents (453066 virtual) 2018-03-28 00:35:41,963 : INFO : 88 batches submitted to accumulate stats from 5632 documents (458371 virtual) 2018-03-28 00:35:41,978 : INFO : 89 batches submitted to accumulate stats from 5696 documents (463517 virtual) 2018-03-28 00:35:42,092 : INFO : 90 batches submitted to accumulate stats from 5760 documents (469153 virtual) 2018-03-28 00:35:42,137 : INFO : 91 batches submitted to accumulate stats from 5824 documents (474495 virtual) 2018-03-28 00:35:42,150 : INFO : 92 batches submitted to accumulate stats from 5888 documents (479818 virtual) 2018-03-28 00:35:42,234 : INFO : 93 batches submitted to accumulate stats from 5952 documents (485199 virtual) 2018-03-28 00:35:42,288 : INFO : 94 batches submitted to accumulate stats from 6016 documents (491103 virtual) 2018-03-28 00:35:42,327 : INFO : 95 batches submitted to accumulate stats from 6080 documents (496453 virtual) 2018-03-28 00:35:42,397 : INFO : 96 batches submitted to accumulate stats from 6144 documents (501470 virtual) 2018-03-28 00:35:42,465 : INFO : 97 batches submitted to accumulate stats from 6208 documents (506849 virtual) 2018-03-28 00:35:42,483 : INFO : 98 batches submitted to accumulate stats from 6272 documents (511923 virtual) 2018-03-28 00:35:42,601 : INFO : 99 batches submitted to accumulate stats from 6336 documents (516767 virtual) 2018-03-28 00:35:42,643 : INFO : 100 batches submitted to accumulate stats from 6400 documents (521852 virtual) 2018-03-28 00:35:42,648 : INFO : 101 batches submitted to accumulate stats from 6464 documents (526866 virtual) 2018-03-28 00:35:42,757 : INFO : 102 batches submitted to accumulate stats from 6528 documents (532029 virtual) 2018-03-28 00:35:42,792 : INFO : 103 batches submitted to accumulate stats from 6592 documents (537379 virtual) 2018-03-28 00:35:42,810 : INFO : 104 batches submitted to accumulate stats from 6656 documents (542324 virtual) 2018-03-28 00:35:42,910 : INFO : 105 batches submitted to accumulate stats from 6720 documents (547604 virtual) 2018-03-28 00:35:42,953 : INFO : 106 batches submitted to accumulate stats from 6784 documents (552876 virtual) 2018-03-28 00:35:42,987 : INFO : 107 batches submitted to accumulate stats from 6848 documents (557940 virtual) 2018-03-28 00:35:43,078 : INFO : 108 batches submitted to accumulate stats from 6912 documents (563355 virtual) 2018-03-28 00:35:43,118 : INFO : 109 batches submitted to accumulate stats from 6976 documents (568839 virtual) 2018-03-28 00:35:43,124 : INFO : 110 batches submitted to accumulate stats from 7040 documents (573876 virtual) 2018-03-28 00:35:43,248 : INFO : 111 batches submitted to accumulate stats from 7104 documents (578111 virtual) 2018-03-28 00:35:43,282 : INFO : 112 batches submitted to accumulate stats from 7168 documents (583423 virtual) 2018-03-28 00:35:43,293 : INFO : 113 batches submitted to accumulate stats from 7232 documents (588643 virtual) 2018-03-28 00:35:43,401 : INFO : 114 batches submitted to accumulate stats from 7296 documents (593440 virtual) 2018-03-28 00:35:43,434 : INFO : 115 batches submitted to accumulate stats from 7360 documents (599104 virtual) 2018-03-28 00:35:43,469 : INFO : 116 batches submitted to accumulate stats from 7424 documents (604164 virtual) 2018-03-28 00:35:43,540 : INFO : 117 batches submitted to accumulate stats from 7488 documents (609254 virtual) 2018-03-28 00:35:43,603 : INFO : 118 batches submitted to accumulate stats from 7552 documents (614718 virtual) 2018-03-28 00:35:43,646 : INFO : 119 batches submitted to accumulate stats from 7616 documents (619684 virtual) 2018-03-28 00:35:43,741 : INFO : 120 batches submitted to accumulate stats from 7680 documents (624868 virtual) 2018-03-28 00:35:43,747 : INFO : 121 batches submitted to accumulate stats from 7744 documents (630169 virtual) 2018-03-28 00:35:43,784 : INFO : 122 batches submitted to accumulate stats from 7808 documents (635785 virtual) 2018-03-28 00:35:43,906 : INFO : 123 batches submitted to accumulate stats from 7872 documents (640705 virtual) 2018-03-28 00:35:43,920 : INFO : 124 batches submitted to accumulate stats from 7936 documents (646061 virtual) 2018-03-28 00:35:43,946 : INFO : 125 batches submitted to accumulate stats from 8000 documents (647394 virtual) 2018-03-28 00:35:44,159 : INFO : serializing accumulator to return to master... 2018-03-28 00:35:44,197 : INFO : serializing accumulator to return to master... 2018-03-28 00:35:44,165 : INFO : accumulator serialized 2018-03-28 00:35:44,202 : INFO : accumulator serialized 2018-03-28 00:35:44,242 : INFO : serializing accumulator to return to master... 2018-03-28 00:35:44,250 : INFO : accumulator serialized 2018-03-28 00:35:44,300 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:35:44,362 : INFO : accumulated word occurrence stats for 648012 virtual documents 2018-03-28 00:35:44,818 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_20 2018-03-28 00:35:44,820 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_20.expElogbeta.npy with mmap=None 2018-03-28 00:35:44,823 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:35:44,824 : INFO : setting ignored attribute state to None 2018-03-28 00:35:44,824 : INFO : setting ignored attribute id2word to None 2018-03-28 00:35:44,825 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_20 2018-03-28 00:35:44,826 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_20.state 2018-03-28 00:35:44,835 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_20.state 2018-03-28 00:35:44,848 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:35:44,894 : INFO : 1 batches submitted to accumulate stats from 64 documents (4982 virtual) 2018-03-28 00:35:44,901 : INFO : 2 batches submitted to accumulate stats from 128 documents (10157 virtual) 2018-03-28 00:35:44,907 : INFO : 3 batches submitted to accumulate stats from 192 documents (15057 virtual) 2018-03-28 00:35:44,913 : INFO : 4 batches submitted to accumulate stats from 256 documents (19941 virtual) 2018-03-28 00:35:44,920 : INFO : 5 batches submitted to accumulate stats from 320 documents (25048 virtual) 2018-03-28 00:35:44,927 : INFO : 6 batches submitted to accumulate stats from 384 documents (30581 virtual) 2018-03-28 00:35:45,072 : INFO : 7 batches submitted to accumulate stats from 448 documents (35845 virtual) 2018-03-28 00:35:45,087 : INFO : 8 batches submitted to accumulate stats from 512 documents (40777 virtual) 2018-03-28 00:35:45,098 : INFO : 9 batches submitted to accumulate stats from 576 documents (45638 virtual) 2018-03-28 00:35:45,246 : INFO : 10 batches submitted to accumulate stats from 640 documents (51118 virtual) 2018-03-28 00:35:45,258 : INFO : 11 batches submitted to accumulate stats from 704 documents (56407 virtual) 2018-03-28 00:35:45,283 : INFO : 12 batches submitted to accumulate stats from 768 documents (61064 virtual) 2018-03-28 00:35:45,402 : INFO : 13 batches submitted to accumulate stats from 832 documents (66752 virtual) 2018-03-28 00:35:45,427 : INFO : 14 batches submitted to accumulate stats from 896 documents (72410 virtual) 2018-03-28 00:35:45,475 : INFO : 15 batches submitted to accumulate stats from 960 documents (77897 virtual) 2018-03-28 00:35:45,590 : INFO : 16 batches submitted to accumulate stats from 1024 documents (83693 virtual) 2018-03-28 00:35:45,599 : INFO : 17 batches submitted to accumulate stats from 1088 documents (88824 virtual) 2018-03-28 00:35:45,662 : INFO : 18 batches submitted to accumulate stats from 1152 documents (93843 virtual) 2018-03-28 00:35:45,760 : INFO : 19 batches submitted to accumulate stats from 1216 documents (98507 virtual) 2018-03-28 00:35:45,772 : INFO : 20 batches submitted to accumulate stats from 1280 documents (104200 virtual) 2018-03-28 00:35:45,846 : INFO : 21 batches submitted to accumulate stats from 1344 documents (109596 virtual) 2018-03-28 00:35:45,946 : INFO : 22 batches submitted to accumulate stats from 1408 documents (114736 virtual) 2018-03-28 00:35:45,955 : INFO : 23 batches submitted to accumulate stats from 1472 documents (120412 virtual) 2018-03-28 00:35:46,030 : INFO : 24 batches submitted to accumulate stats from 1536 documents (125964 virtual) 2018-03-28 00:35:46,107 : INFO : 25 batches submitted to accumulate stats from 1600 documents (131215 virtual) 2018-03-28 00:35:46,128 : INFO : 26 batches submitted to accumulate stats from 1664 documents (135773 virtual) 2018-03-28 00:35:46,203 : INFO : 27 batches submitted to accumulate stats from 1728 documents (141410 virtual) 2018-03-28 00:35:46,281 : INFO : 28 batches submitted to accumulate stats from 1792 documents (146517 virtual) 2018-03-28 00:35:46,320 : INFO : 29 batches submitted to accumulate stats from 1856 documents (152359 virtual) 2018-03-28 00:35:46,383 : INFO : 30 batches submitted to accumulate stats from 1920 documents (157670 virtual) 2018-03-28 00:35:46,451 : INFO : 31 batches submitted to accumulate stats from 1984 documents (162805 virtual) 2018-03-28 00:35:46,496 : INFO : 32 batches submitted to accumulate stats from 2048 documents (167775 virtual) 2018-03-28 00:35:46,585 : INFO : 33 batches submitted to accumulate stats from 2112 documents (172403 virtual) 2018-03-28 00:35:46,604 : INFO : 34 batches submitted to accumulate stats from 2176 documents (177998 virtual) 2018-03-28 00:35:46,687 : INFO : 35 batches submitted to accumulate stats from 2240 documents (183591 virtual) 2018-03-28 00:35:46,768 : INFO : 36 batches submitted to accumulate stats from 2304 documents (188745 virtual) 2018-03-28 00:35:46,799 : INFO : 37 batches submitted to accumulate stats from 2368 documents (194385 virtual) 2018-03-28 00:35:46,855 : INFO : 38 batches submitted to accumulate stats from 2432 documents (200561 virtual) 2018-03-28 00:35:46,917 : INFO : 39 batches submitted to accumulate stats from 2496 documents (205501 virtual) 2018-03-28 00:35:46,976 : INFO : 40 batches submitted to accumulate stats from 2560 documents (210729 virtual) 2018-03-28 00:35:47,030 : INFO : 41 batches submitted to accumulate stats from 2624 documents (215226 virtual) 2018-03-28 00:35:47,111 : INFO : 42 batches submitted to accumulate stats from 2688 documents (220560 virtual) 2018-03-28 00:35:47,165 : INFO : 43 batches submitted to accumulate stats from 2752 documents (225093 virtual) 2018-03-28 00:35:47,208 : INFO : 44 batches submitted to accumulate stats from 2816 documents (229832 virtual) 2018-03-28 00:35:47,280 : INFO : 45 batches submitted to accumulate stats from 2880 documents (235373 virtual) 2018-03-28 00:35:47,343 : INFO : 46 batches submitted to accumulate stats from 2944 documents (240140 virtual) 2018-03-28 00:35:47,361 : INFO : 47 batches submitted to accumulate stats from 3008 documents (244952 virtual) 2018-03-28 00:35:47,481 : INFO : 48 batches submitted to accumulate stats from 3072 documents (249877 virtual) 2018-03-28 00:35:47,487 : INFO : 49 batches submitted to accumulate stats from 3136 documents (254875 virtual) 2018-03-28 00:35:47,528 : INFO : 50 batches submitted to accumulate stats from 3200 documents (260445 virtual) 2018-03-28 00:35:47,646 : INFO : 51 batches submitted to accumulate stats from 3264 documents (265513 virtual) 2018-03-28 00:35:47,668 : INFO : 52 batches submitted to accumulate stats from 3328 documents (270743 virtual) 2018-03-28 00:35:47,685 : INFO : 53 batches submitted to accumulate stats from 3392 documents (275753 virtual) 2018-03-28 00:35:47,806 : INFO : 54 batches submitted to accumulate stats from 3456 documents (280784 virtual) 2018-03-28 00:35:47,834 : INFO : 55 batches submitted to accumulate stats from 3520 documents (286064 virtual) 2018-03-28 00:35:47,882 : INFO : 56 batches submitted to accumulate stats from 3584 documents (290769 virtual) 2018-03-28 00:35:47,968 : INFO : 57 batches submitted to accumulate stats from 3648 documents (296488 virtual) 2018-03-28 00:35:48,027 : INFO : 58 batches submitted to accumulate stats from 3712 documents (301421 virtual) 2018-03-28 00:35:48,033 : INFO : 59 batches submitted to accumulate stats from 3776 documents (306734 virtual) 2018-03-28 00:35:48,151 : INFO : 60 batches submitted to accumulate stats from 3840 documents (311918 virtual) 2018-03-28 00:35:48,190 : INFO : 61 batches submitted to accumulate stats from 3904 documents (317212 virtual) 2018-03-28 00:35:48,196 : INFO : 62 batches submitted to accumulate stats from 3968 documents (322089 virtual) 2018-03-28 00:35:48,347 : INFO : 63 batches submitted to accumulate stats from 4032 documents (326781 virtual) 2018-03-28 00:35:48,355 : INFO : 64 batches submitted to accumulate stats from 4096 documents (332204 virtual) 2018-03-28 00:35:48,374 : INFO : 65 batches submitted to accumulate stats from 4160 documents (337312 virtual) 2018-03-28 00:35:48,525 : INFO : 66 batches submitted to accumulate stats from 4224 documents (342640 virtual) 2018-03-28 00:35:48,556 : INFO : 67 batches submitted to accumulate stats from 4288 documents (347711 virtual) 2018-03-28 00:35:48,568 : INFO : 68 batches submitted to accumulate stats from 4352 documents (352897 virtual) 2018-03-28 00:35:48,704 : INFO : 69 batches submitted to accumulate stats from 4416 documents (358011 virtual) 2018-03-28 00:35:48,752 : INFO : 70 batches submitted to accumulate stats from 4480 documents (363147 virtual) 2018-03-28 00:35:48,758 : INFO : 71 batches submitted to accumulate stats from 4544 documents (368013 virtual) 2018-03-28 00:35:48,889 : INFO : 72 batches submitted to accumulate stats from 4608 documents (373143 virtual) 2018-03-28 00:35:48,908 : INFO : 73 batches submitted to accumulate stats from 4672 documents (378189 virtual) 2018-03-28 00:35:48,956 : INFO : 74 batches submitted to accumulate stats from 4736 documents (383853 virtual) 2018-03-28 00:35:49,059 : INFO : 75 batches submitted to accumulate stats from 4800 documents (389123 virtual) 2018-03-28 00:35:49,080 : INFO : 76 batches submitted to accumulate stats from 4864 documents (393457 virtual) 2018-03-28 00:35:49,108 : INFO : 77 batches submitted to accumulate stats from 4928 documents (398948 virtual) 2018-03-28 00:35:49,233 : INFO : 78 batches submitted to accumulate stats from 4992 documents (403866 virtual) 2018-03-28 00:35:49,257 : INFO : 79 batches submitted to accumulate stats from 5056 documents (408996 virtual) 2018-03-28 00:35:49,283 : INFO : 80 batches submitted to accumulate stats from 5120 documents (414464 virtual) 2018-03-28 00:35:49,412 : INFO : 81 batches submitted to accumulate stats from 5184 documents (420250 virtual) 2018-03-28 00:35:49,423 : INFO : 82 batches submitted to accumulate stats from 5248 documents (425365 virtual) 2018-03-28 00:35:49,460 : INFO : 83 batches submitted to accumulate stats from 5312 documents (430578 virtual) 2018-03-28 00:35:49,578 : INFO : 84 batches submitted to accumulate stats from 5376 documents (436352 virtual) 2018-03-28 00:35:49,617 : INFO : 85 batches submitted to accumulate stats from 5440 documents (441886 virtual) 2018-03-28 00:35:49,623 : INFO : 86 batches submitted to accumulate stats from 5504 documents (447523 virtual) 2018-03-28 00:35:49,768 : INFO : 87 batches submitted to accumulate stats from 5568 documents (452242 virtual) 2018-03-28 00:35:49,797 : INFO : 88 batches submitted to accumulate stats from 5632 documents (457553 virtual) 2018-03-28 00:35:49,819 : INFO : 89 batches submitted to accumulate stats from 5696 documents (463000 virtual) 2018-03-28 00:35:49,914 : INFO : 90 batches submitted to accumulate stats from 5760 documents (468341 virtual) 2018-03-28 00:35:49,998 : INFO : 91 batches submitted to accumulate stats from 5824 documents (473805 virtual) 2018-03-28 00:35:50,036 : INFO : 92 batches submitted to accumulate stats from 5888 documents (479144 virtual) 2018-03-28 00:35:50,065 : INFO : 93 batches submitted to accumulate stats from 5952 documents (484277 virtual) 2018-03-28 00:35:50,182 : INFO : 94 batches submitted to accumulate stats from 6016 documents (490100 virtual) 2018-03-28 00:35:50,214 : INFO : 95 batches submitted to accumulate stats from 6080 documents (495729 virtual) 2018-03-28 00:35:50,240 : INFO : 96 batches submitted to accumulate stats from 6144 documents (500712 virtual) 2018-03-28 00:35:50,363 : INFO : 97 batches submitted to accumulate stats from 6208 documents (506253 virtual) 2018-03-28 00:35:50,406 : INFO : 98 batches submitted to accumulate stats from 6272 documents (511296 virtual) 2018-03-28 00:35:50,417 : INFO : 99 batches submitted to accumulate stats from 6336 documents (516045 virtual) 2018-03-28 00:35:50,525 : INFO : 100 batches submitted to accumulate stats from 6400 documents (521206 virtual) 2018-03-28 00:35:50,590 : INFO : 101 batches submitted to accumulate stats from 6464 documents (526237 virtual) 2018-03-28 00:35:50,630 : INFO : 102 batches submitted to accumulate stats from 6528 documents (531188 virtual) 2018-03-28 00:35:50,677 : INFO : 103 batches submitted to accumulate stats from 6592 documents (536374 virtual) 2018-03-28 00:35:50,791 : INFO : 104 batches submitted to accumulate stats from 6656 documents (541574 virtual) 2018-03-28 00:35:50,802 : INFO : 105 batches submitted to accumulate stats from 6720 documents (546747 virtual) 2018-03-28 00:35:50,833 : INFO : 106 batches submitted to accumulate stats from 6784 documents (551868 virtual) 2018-03-28 00:35:50,944 : INFO : 107 batches submitted to accumulate stats from 6848 documents (557264 virtual) 2018-03-28 00:35:50,987 : INFO : 108 batches submitted to accumulate stats from 6912 documents (562641 virtual) 2018-03-28 00:35:51,000 : INFO : 109 batches submitted to accumulate stats from 6976 documents (568059 virtual) 2018-03-28 00:35:51,110 : INFO : 110 batches submitted to accumulate stats from 7040 documents (573099 virtual) 2018-03-28 00:35:51,158 : INFO : 111 batches submitted to accumulate stats from 7104 documents (577546 virtual) 2018-03-28 00:35:51,184 : INFO : 112 batches submitted to accumulate stats from 7168 documents (582449 virtual) 2018-03-28 00:35:51,289 : INFO : 113 batches submitted to accumulate stats from 7232 documents (587997 virtual) 2018-03-28 00:35:51,346 : INFO : 114 batches submitted to accumulate stats from 7296 documents (592649 virtual) 2018-03-28 00:35:51,363 : INFO : 115 batches submitted to accumulate stats from 7360 documents (598365 virtual) 2018-03-28 00:35:51,470 : INFO : 116 batches submitted to accumulate stats from 7424 documents (603690 virtual) 2018-03-28 00:35:51,484 : INFO : 117 batches submitted to accumulate stats from 7488 documents (608587 virtual) 2018-03-28 00:35:51,540 : INFO : 118 batches submitted to accumulate stats from 7552 documents (613868 virtual) 2018-03-28 00:35:51,647 : INFO : 119 batches submitted to accumulate stats from 7616 documents (618819 virtual) 2018-03-28 00:35:51,667 : INFO : 120 batches submitted to accumulate stats from 7680 documents (624121 virtual) 2018-03-28 00:35:51,726 : INFO : 121 batches submitted to accumulate stats from 7744 documents (629269 virtual) 2018-03-28 00:35:51,810 : INFO : 122 batches submitted to accumulate stats from 7808 documents (635072 virtual) 2018-03-28 00:35:51,837 : INFO : 123 batches submitted to accumulate stats from 7872 documents (640050 virtual) 2018-03-28 00:35:51,927 : INFO : 124 batches submitted to accumulate stats from 7936 documents (645205 virtual) 2018-03-28 00:35:51,968 : INFO : 125 batches submitted to accumulate stats from 8000 documents (647463 virtual) 2018-03-28 00:35:52,191 : INFO : serializing accumulator to return to master... 2018-03-28 00:35:52,225 : INFO : serializing accumulator to return to master... 2018-03-28 00:35:52,199 : INFO : accumulator serialized 2018-03-28 00:35:52,266 : INFO : serializing accumulator to return to master... 2018-03-28 00:35:52,229 : INFO : accumulator serialized 2018-03-28 00:35:52,275 : INFO : accumulator serialized 2018-03-28 00:35:52,358 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:35:52,443 : INFO : accumulated word occurrence stats for 648093 virtual documents 2018-03-28 00:35:53,009 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_25 2018-03-28 00:35:53,011 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-28 00:35:53,013 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:35:53,014 : INFO : setting ignored attribute state to None 2018-03-28 00:35:53,015 : INFO : setting ignored attribute id2word to None 2018-03-28 00:35:53,015 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_25 2018-03-28 00:35:53,016 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_25.state 2018-03-28 00:35:53,028 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_25.state 2018-03-28 00:35:53,043 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:35:53,089 : INFO : 1 batches submitted to accumulate stats from 64 documents (4982 virtual) 2018-03-28 00:35:53,097 : INFO : 2 batches submitted to accumulate stats from 128 documents (10157 virtual) 2018-03-28 00:35:53,103 : INFO : 3 batches submitted to accumulate stats from 192 documents (15057 virtual) 2018-03-28 00:35:53,111 : INFO : 4 batches submitted to accumulate stats from 256 documents (19941 virtual) 2018-03-28 00:35:53,118 : INFO : 5 batches submitted to accumulate stats from 320 documents (25048 virtual) 2018-03-28 00:35:53,125 : INFO : 6 batches submitted to accumulate stats from 384 documents (30581 virtual) 2018-03-28 00:35:53,281 : INFO : 7 batches submitted to accumulate stats from 448 documents (35845 virtual) 2018-03-28 00:35:53,298 : INFO : 8 batches submitted to accumulate stats from 512 documents (40777 virtual) 2018-03-28 00:35:53,332 : INFO : 9 batches submitted to accumulate stats from 576 documents (45638 virtual) 2018-03-28 00:35:53,485 : INFO : 10 batches submitted to accumulate stats from 640 documents (51118 virtual) 2018-03-28 00:35:53,496 : INFO : 11 batches submitted to accumulate stats from 704 documents (56407 virtual) 2018-03-28 00:35:53,538 : INFO : 12 batches submitted to accumulate stats from 768 documents (61056 virtual) 2018-03-28 00:35:53,678 : INFO : 13 batches submitted to accumulate stats from 832 documents (66787 virtual) 2018-03-28 00:35:53,701 : INFO : 14 batches submitted to accumulate stats from 896 documents (72403 virtual) 2018-03-28 00:35:53,722 : INFO : 15 batches submitted to accumulate stats from 960 documents (77560 virtual) 2018-03-28 00:35:53,883 : INFO : 16 batches submitted to accumulate stats from 1024 documents (83461 virtual) 2018-03-28 00:35:53,904 : INFO : 17 batches submitted to accumulate stats from 1088 documents (88623 virtual) 2018-03-28 00:35:53,911 : INFO : 18 batches submitted to accumulate stats from 1152 documents (93573 virtual) 2018-03-28 00:35:54,080 : INFO : 19 batches submitted to accumulate stats from 1216 documents (98302 virtual) 2018-03-28 00:35:54,106 : INFO : 20 batches submitted to accumulate stats from 1280 documents (104024 virtual) 2018-03-28 00:35:54,131 : INFO : 21 batches submitted to accumulate stats from 1344 documents (109331 virtual) 2018-03-28 00:35:54,277 : INFO : 22 batches submitted to accumulate stats from 1408 documents (114598 virtual) 2018-03-28 00:35:54,292 : INFO : 23 batches submitted to accumulate stats from 1472 documents (120281 virtual) 2018-03-28 00:35:54,333 : INFO : 24 batches submitted to accumulate stats from 1536 documents (125702 virtual) 2018-03-28 00:35:54,459 : INFO : 25 batches submitted to accumulate stats from 1600 documents (130886 virtual) 2018-03-28 00:35:54,503 : INFO : 26 batches submitted to accumulate stats from 1664 documents (135655 virtual) 2018-03-28 00:35:54,536 : INFO : 27 batches submitted to accumulate stats from 1728 documents (141221 virtual) 2018-03-28 00:35:54,652 : INFO : 28 batches submitted to accumulate stats from 1792 documents (146417 virtual) 2018-03-28 00:35:54,718 : INFO : 29 batches submitted to accumulate stats from 1856 documents (152022 virtual) 2018-03-28 00:35:54,733 : INFO : 30 batches submitted to accumulate stats from 1920 documents (157433 virtual) 2018-03-28 00:35:54,830 : INFO : 31 batches submitted to accumulate stats from 1984 documents (162621 virtual) 2018-03-28 00:35:54,913 : INFO : 32 batches submitted to accumulate stats from 2048 documents (167492 virtual) 2018-03-28 00:35:54,926 : INFO : 33 batches submitted to accumulate stats from 2112 documents (172332 virtual) 2018-03-28 00:35:55,038 : INFO : 34 batches submitted to accumulate stats from 2176 documents (177761 virtual) 2018-03-28 00:35:55,116 : INFO : 35 batches submitted to accumulate stats from 2240 documents (183357 virtual) 2018-03-28 00:35:55,135 : INFO : 36 batches submitted to accumulate stats from 2304 documents (188436 virtual) 2018-03-28 00:35:55,255 : INFO : 37 batches submitted to accumulate stats from 2368 documents (194151 virtual) 2018-03-28 00:35:55,289 : INFO : 38 batches submitted to accumulate stats from 2432 documents (200474 virtual) 2018-03-28 00:35:55,310 : INFO : 39 batches submitted to accumulate stats from 2496 documents (205373 virtual) 2018-03-28 00:35:55,430 : INFO : 40 batches submitted to accumulate stats from 2560 documents (210566 virtual) 2018-03-28 00:35:55,476 : INFO : 41 batches submitted to accumulate stats from 2624 documents (215073 virtual) 2018-03-28 00:35:55,540 : INFO : 42 batches submitted to accumulate stats from 2688 documents (220401 virtual) 2018-03-28 00:35:55,622 : INFO : 43 batches submitted to accumulate stats from 2752 documents (224827 virtual) 2018-03-28 00:35:55,707 : INFO : 44 batches submitted to accumulate stats from 2816 documents (229446 virtual) 2018-03-28 00:35:55,748 : INFO : 45 batches submitted to accumulate stats from 2880 documents (235053 virtual) 2018-03-28 00:35:55,811 : INFO : 46 batches submitted to accumulate stats from 2944 documents (239930 virtual) 2018-03-28 00:35:55,882 : INFO : 47 batches submitted to accumulate stats from 3008 documents (244676 virtual) 2018-03-28 00:35:55,953 : INFO : 48 batches submitted to accumulate stats from 3072 documents (249674 virtual) 2018-03-28 00:35:55,968 : INFO : 49 batches submitted to accumulate stats from 3136 documents (254623 virtual) 2018-03-28 00:35:56,066 : INFO : 50 batches submitted to accumulate stats from 3200 documents (260221 virtual) 2018-03-28 00:35:56,153 : INFO : 51 batches submitted to accumulate stats from 3264 documents (265277 virtual) 2018-03-28 00:35:56,160 : INFO : 52 batches submitted to accumulate stats from 3328 documents (270454 virtual) 2018-03-28 00:35:56,233 : INFO : 53 batches submitted to accumulate stats from 3392 documents (275654 virtual) 2018-03-28 00:35:56,329 : INFO : 54 batches submitted to accumulate stats from 3456 documents (280696 virtual) 2018-03-28 00:35:56,353 : INFO : 55 batches submitted to accumulate stats from 3520 documents (285590 virtual) 2018-03-28 00:35:56,428 : INFO : 56 batches submitted to accumulate stats from 3584 documents (290491 virtual) 2018-03-28 00:35:56,530 : INFO : 57 batches submitted to accumulate stats from 3648 documents (296198 virtual) 2018-03-28 00:35:56,549 : INFO : 58 batches submitted to accumulate stats from 3712 documents (301241 virtual) 2018-03-28 00:35:56,611 : INFO : 59 batches submitted to accumulate stats from 3776 documents (306370 virtual) 2018-03-28 00:35:56,712 : INFO : 60 batches submitted to accumulate stats from 3840 documents (311632 virtual) 2018-03-28 00:35:56,755 : INFO : 61 batches submitted to accumulate stats from 3904 documents (316831 virtual) 2018-03-28 00:35:56,811 : INFO : 62 batches submitted to accumulate stats from 3968 documents (321771 virtual) 2018-03-28 00:35:56,890 : INFO : 63 batches submitted to accumulate stats from 4032 documents (326505 virtual) 2018-03-28 00:35:56,960 : INFO : 64 batches submitted to accumulate stats from 4096 documents (331876 virtual) 2018-03-28 00:35:57,010 : INFO : 65 batches submitted to accumulate stats from 4160 documents (337046 virtual) 2018-03-28 00:35:57,091 : INFO : 66 batches submitted to accumulate stats from 4224 documents (342312 virtual) 2018-03-28 00:35:57,172 : INFO : 67 batches submitted to accumulate stats from 4288 documents (347404 virtual) 2018-03-28 00:35:57,224 : INFO : 68 batches submitted to accumulate stats from 4352 documents (352519 virtual) 2018-03-28 00:35:57,249 : INFO : 69 batches submitted to accumulate stats from 4416 documents (357694 virtual) 2018-03-28 00:35:57,345 : INFO : 70 batches submitted to accumulate stats from 4480 documents (362903 virtual) 2018-03-28 00:35:57,432 : INFO : 71 batches submitted to accumulate stats from 4544 documents (367575 virtual) 2018-03-28 00:35:57,459 : INFO : 72 batches submitted to accumulate stats from 4608 documents (372807 virtual) 2018-03-28 00:35:57,522 : INFO : 73 batches submitted to accumulate stats from 4672 documents (377997 virtual) 2018-03-28 00:35:57,620 : INFO : 74 batches submitted to accumulate stats from 4736 documents (383571 virtual) 2018-03-28 00:35:57,672 : INFO : 75 batches submitted to accumulate stats from 4800 documents (388804 virtual) 2018-03-28 00:35:57,722 : INFO : 76 batches submitted to accumulate stats from 4864 documents (393255 virtual) 2018-03-28 00:35:57,789 : INFO : 77 batches submitted to accumulate stats from 4928 documents (398540 virtual) 2018-03-28 00:35:57,876 : INFO : 78 batches submitted to accumulate stats from 4992 documents (403620 virtual) 2018-03-28 00:35:57,901 : INFO : 79 batches submitted to accumulate stats from 5056 documents (408677 virtual) 2018-03-28 00:35:57,996 : INFO : 80 batches submitted to accumulate stats from 5120 documents (414061 virtual) 2018-03-28 00:35:58,064 : INFO : 81 batches submitted to accumulate stats from 5184 documents (419596 virtual) 2018-03-28 00:35:58,093 : INFO : 82 batches submitted to accumulate stats from 5248 documents (424836 virtual) 2018-03-28 00:35:58,191 : INFO : 83 batches submitted to accumulate stats from 5312 documents (430133 virtual) 2018-03-28 00:35:58,246 : INFO : 84 batches submitted to accumulate stats from 5376 documents (435883 virtual) 2018-03-28 00:35:58,271 : INFO : 85 batches submitted to accumulate stats from 5440 documents (441434 virtual) 2018-03-28 00:35:58,401 : INFO : 86 batches submitted to accumulate stats from 5504 documents (447053 virtual) 2018-03-28 00:35:58,429 : INFO : 87 batches submitted to accumulate stats from 5568 documents (451735 virtual) 2018-03-28 00:35:58,465 : INFO : 88 batches submitted to accumulate stats from 5632 documents (457192 virtual) 2018-03-28 00:35:58,613 : INFO : 89 batches submitted to accumulate stats from 5696 documents (462355 virtual) 2018-03-28 00:35:58,649 : INFO : 90 batches submitted to accumulate stats from 5760 documents (467655 virtual) 2018-03-28 00:35:58,659 : INFO : 91 batches submitted to accumulate stats from 5824 documents (473056 virtual) 2018-03-28 00:35:58,806 : INFO : 92 batches submitted to accumulate stats from 5888 documents (478655 virtual) 2018-03-28 00:35:58,832 : INFO : 93 batches submitted to accumulate stats from 5952 documents (483934 virtual) 2018-03-28 00:35:58,859 : INFO : 94 batches submitted to accumulate stats from 6016 documents (489508 virtual) 2018-03-28 00:35:59,014 : INFO : 95 batches submitted to accumulate stats from 6080 documents (494980 virtual) 2018-03-28 00:35:59,020 : INFO : 96 batches submitted to accumulate stats from 6144 documents (500293 virtual) 2018-03-28 00:35:59,065 : INFO : 97 batches submitted to accumulate stats from 6208 documents (505658 virtual) 2018-03-28 00:35:59,214 : INFO : 98 batches submitted to accumulate stats from 6272 documents (510711 virtual) 2018-03-28 00:35:59,228 : INFO : 99 batches submitted to accumulate stats from 6336 documents (515435 virtual) 2018-03-28 00:35:59,283 : INFO : 100 batches submitted to accumulate stats from 6400 documents (520602 virtual) 2018-03-28 00:35:59,396 : INFO : 101 batches submitted to accumulate stats from 6464 documents (525661 virtual) 2018-03-28 00:35:59,436 : INFO : 102 batches submitted to accumulate stats from 6528 documents (530650 virtual) 2018-03-28 00:35:59,484 : INFO : 103 batches submitted to accumulate stats from 6592 documents (535725 virtual) 2018-03-28 00:35:59,602 : INFO : 104 batches submitted to accumulate stats from 6656 documents (540936 virtual) 2018-03-28 00:35:59,641 : INFO : 105 batches submitted to accumulate stats from 6720 documents (546058 virtual) 2018-03-28 00:35:59,704 : INFO : 106 batches submitted to accumulate stats from 6784 documents (551013 virtual) 2018-03-28 00:35:59,817 : INFO : 107 batches submitted to accumulate stats from 6848 documents (556507 virtual) 2018-03-28 00:35:59,860 : INFO : 108 batches submitted to accumulate stats from 6912 documents (561629 virtual) 2018-03-28 00:35:59,899 : INFO : 109 batches submitted to accumulate stats from 6976 documents (567108 virtual) 2018-03-28 00:35:59,999 : INFO : 110 batches submitted to accumulate stats from 7040 documents (572121 virtual) 2018-03-28 00:36:00,069 : INFO : 111 batches submitted to accumulate stats from 7104 documents (576719 virtual) 2018-03-28 00:36:00,089 : INFO : 112 batches submitted to accumulate stats from 7168 documents (581446 virtual) 2018-03-28 00:36:00,193 : INFO : 113 batches submitted to accumulate stats from 7232 documents (587049 virtual) 2018-03-28 00:36:00,263 : INFO : 114 batches submitted to accumulate stats from 7296 documents (591835 virtual) 2018-03-28 00:36:00,306 : INFO : 115 batches submitted to accumulate stats from 7360 documents (597115 virtual) 2018-03-28 00:36:00,396 : INFO : 116 batches submitted to accumulate stats from 7424 documents (602688 virtual) 2018-03-28 00:36:00,428 : INFO : 117 batches submitted to accumulate stats from 7488 documents (607592 virtual) 2018-03-28 00:36:00,497 : INFO : 118 batches submitted to accumulate stats from 7552 documents (612613 virtual) 2018-03-28 00:36:00,614 : INFO : 119 batches submitted to accumulate stats from 7616 documents (617826 virtual) 2018-03-28 00:36:00,620 : INFO : 120 batches submitted to accumulate stats from 7680 documents (623064 virtual) 2018-03-28 00:36:00,689 : INFO : 121 batches submitted to accumulate stats from 7744 documents (628311 virtual) 2018-03-28 00:36:00,785 : INFO : 122 batches submitted to accumulate stats from 7808 documents (634180 virtual) 2018-03-28 00:36:00,819 : INFO : 123 batches submitted to accumulate stats from 7872 documents (639156 virtual) 2018-03-28 00:36:00,912 : INFO : 124 batches submitted to accumulate stats from 7936 documents (644213 virtual) 2018-03-28 00:36:00,962 : INFO : 125 batches submitted to accumulate stats from 8000 documents (647458 virtual) 2018-03-28 00:36:01,244 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:01,266 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:01,269 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:01,248 : INFO : accumulator serialized 2018-03-28 00:36:01,278 : INFO : accumulator serialized 2018-03-28 00:36:01,272 : INFO : accumulator serialized 2018-03-28 00:36:01,417 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:36:01,553 : INFO : accumulated word occurrence stats for 648117 virtual documents 2018-03-28 00:36:02,238 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_30 2018-03-28 00:36:02,239 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-28 00:36:02,243 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:36:02,244 : INFO : setting ignored attribute state to None 2018-03-28 00:36:02,245 : INFO : setting ignored attribute id2word to None 2018-03-28 00:36:02,245 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_30 2018-03-28 00:36:02,246 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_30.state 2018-03-28 00:36:02,263 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_30.state 2018-03-28 00:36:02,281 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:36:02,333 : INFO : 1 batches submitted to accumulate stats from 64 documents (4982 virtual) 2018-03-28 00:36:02,341 : INFO : 2 batches submitted to accumulate stats from 128 documents (10157 virtual) 2018-03-28 00:36:02,349 : INFO : 3 batches submitted to accumulate stats from 192 documents (15057 virtual) 2018-03-28 00:36:02,357 : INFO : 4 batches submitted to accumulate stats from 256 documents (19941 virtual) 2018-03-28 00:36:02,364 : INFO : 5 batches submitted to accumulate stats from 320 documents (25048 virtual) 2018-03-28 00:36:02,371 : INFO : 6 batches submitted to accumulate stats from 384 documents (30581 virtual) 2018-03-28 00:36:02,541 : INFO : 7 batches submitted to accumulate stats from 448 documents (35845 virtual) 2018-03-28 00:36:02,554 : INFO : 8 batches submitted to accumulate stats from 512 documents (40777 virtual) 2018-03-28 00:36:02,583 : INFO : 9 batches submitted to accumulate stats from 576 documents (45638 virtual) 2018-03-28 00:36:02,744 : INFO : 10 batches submitted to accumulate stats from 640 documents (51118 virtual) 2018-03-28 00:36:02,764 : INFO : 11 batches submitted to accumulate stats from 704 documents (56333 virtual) 2018-03-28 00:36:02,789 : INFO : 12 batches submitted to accumulate stats from 768 documents (60884 virtual) 2018-03-28 00:36:02,935 : INFO : 13 batches submitted to accumulate stats from 832 documents (66653 virtual) 2018-03-28 00:36:02,977 : INFO : 14 batches submitted to accumulate stats from 896 documents (72104 virtual) 2018-03-28 00:36:02,988 : INFO : 15 batches submitted to accumulate stats from 960 documents (77395 virtual) 2018-03-28 00:36:03,134 : INFO : 16 batches submitted to accumulate stats from 1024 documents (83476 virtual) 2018-03-28 00:36:03,174 : INFO : 17 batches submitted to accumulate stats from 1088 documents (88506 virtual) 2018-03-28 00:36:03,207 : INFO : 18 batches submitted to accumulate stats from 1152 documents (93382 virtual) 2018-03-28 00:36:03,341 : INFO : 19 batches submitted to accumulate stats from 1216 documents (98195 virtual) 2018-03-28 00:36:03,366 : INFO : 20 batches submitted to accumulate stats from 1280 documents (103859 virtual) 2018-03-28 00:36:03,443 : INFO : 21 batches submitted to accumulate stats from 1344 documents (109190 virtual) 2018-03-28 00:36:03,557 : INFO : 22 batches submitted to accumulate stats from 1408 documents (114487 virtual) 2018-03-28 00:36:03,574 : INFO : 23 batches submitted to accumulate stats from 1472 documents (120192 virtual) 2018-03-28 00:36:03,621 : INFO : 24 batches submitted to accumulate stats from 1536 documents (125318 virtual) 2018-03-28 00:36:03,754 : INFO : 25 batches submitted to accumulate stats from 1600 documents (130728 virtual) 2018-03-28 00:36:03,775 : INFO : 26 batches submitted to accumulate stats from 1664 documents (135604 virtual) 2018-03-28 00:36:03,849 : INFO : 27 batches submitted to accumulate stats from 1728 documents (141107 virtual) 2018-03-28 00:36:03,950 : INFO : 28 batches submitted to accumulate stats from 1792 documents (146233 virtual) 2018-03-28 00:36:03,992 : INFO : 29 batches submitted to accumulate stats from 1856 documents (151723 virtual) 2018-03-28 00:36:04,051 : INFO : 30 batches submitted to accumulate stats from 1920 documents (157313 virtual) 2018-03-28 00:36:04,155 : INFO : 31 batches submitted to accumulate stats from 1984 documents (162270 virtual) 2018-03-28 00:36:04,196 : INFO : 32 batches submitted to accumulate stats from 2048 documents (167221 virtual) 2018-03-28 00:36:04,252 : INFO : 33 batches submitted to accumulate stats from 2112 documents (172027 virtual) 2018-03-28 00:36:04,341 : INFO : 34 batches submitted to accumulate stats from 2176 documents (177615 virtual) 2018-03-28 00:36:04,424 : INFO : 35 batches submitted to accumulate stats from 2240 documents (183248 virtual) 2018-03-28 00:36:04,484 : INFO : 36 batches submitted to accumulate stats from 2304 documents (188186 virtual) 2018-03-28 00:36:04,528 : INFO : 37 batches submitted to accumulate stats from 2368 documents (193893 virtual) 2018-03-28 00:36:04,626 : INFO : 38 batches submitted to accumulate stats from 2432 documents (200219 virtual) 2018-03-28 00:36:04,681 : INFO : 39 batches submitted to accumulate stats from 2496 documents (205228 virtual) 2018-03-28 00:36:04,744 : INFO : 40 batches submitted to accumulate stats from 2560 documents (210271 virtual) 2018-03-28 00:36:04,811 : INFO : 41 batches submitted to accumulate stats from 2624 documents (214964 virtual) 2018-03-28 00:36:04,873 : INFO : 42 batches submitted to accumulate stats from 2688 documents (220057 virtual) 2018-03-28 00:36:04,979 : INFO : 43 batches submitted to accumulate stats from 2752 documents (224751 virtual) 2018-03-28 00:36:05,049 : INFO : 44 batches submitted to accumulate stats from 2816 documents (229295 virtual) 2018-03-28 00:36:05,067 : INFO : 45 batches submitted to accumulate stats from 2880 documents (234871 virtual) 2018-03-28 00:36:05,169 : INFO : 46 batches submitted to accumulate stats from 2944 documents (239789 virtual) 2018-03-28 00:36:05,228 : INFO : 47 batches submitted to accumulate stats from 3008 documents (244548 virtual) 2018-03-28 00:36:05,264 : INFO : 48 batches submitted to accumulate stats from 3072 documents (249553 virtual) 2018-03-28 00:36:05,383 : INFO : 49 batches submitted to accumulate stats from 3136 documents (254401 virtual) 2018-03-28 00:36:05,409 : INFO : 50 batches submitted to accumulate stats from 3200 documents (260033 virtual) 2018-03-28 00:36:05,448 : INFO : 51 batches submitted to accumulate stats from 3264 documents (265007 virtual) 2018-03-28 00:36:05,581 : INFO : 52 batches submitted to accumulate stats from 3328 documents (270346 virtual) 2018-03-28 00:36:05,601 : INFO : 53 batches submitted to accumulate stats from 3392 documents (275486 virtual) 2018-03-28 00:36:05,648 : INFO : 54 batches submitted to accumulate stats from 3456 documents (280523 virtual) 2018-03-28 00:36:05,763 : INFO : 55 batches submitted to accumulate stats from 3520 documents (285462 virtual) 2018-03-28 00:36:05,825 : INFO : 56 batches submitted to accumulate stats from 3584 documents (290399 virtual) 2018-03-28 00:36:05,849 : INFO : 57 batches submitted to accumulate stats from 3648 documents (295784 virtual) 2018-03-28 00:36:05,961 : INFO : 58 batches submitted to accumulate stats from 3712 documents (301120 virtual) 2018-03-28 00:36:06,018 : INFO : 59 batches submitted to accumulate stats from 3776 documents (306316 virtual) 2018-03-28 00:36:06,033 : INFO : 60 batches submitted to accumulate stats from 3840 documents (311555 virtual) 2018-03-28 00:36:06,160 : INFO : 61 batches submitted to accumulate stats from 3904 documents (316771 virtual) 2018-03-28 00:36:06,217 : INFO : 62 batches submitted to accumulate stats from 3968 documents (321597 virtual) 2018-03-28 00:36:06,241 : INFO : 63 batches submitted to accumulate stats from 4032 documents (326307 virtual) 2018-03-28 00:36:06,355 : INFO : 64 batches submitted to accumulate stats from 4096 documents (331751 virtual) 2018-03-28 00:36:06,433 : INFO : 65 batches submitted to accumulate stats from 4160 documents (336932 virtual) 2018-03-28 00:36:06,439 : INFO : 66 batches submitted to accumulate stats from 4224 documents (342094 virtual) 2018-03-28 00:36:06,559 : INFO : 67 batches submitted to accumulate stats from 4288 documents (347187 virtual) 2018-03-28 00:36:06,592 : INFO : 68 batches submitted to accumulate stats from 4352 documents (352221 virtual) 2018-03-28 00:36:06,662 : INFO : 69 batches submitted to accumulate stats from 4416 documents (357294 virtual) 2018-03-28 00:36:06,767 : INFO : 70 batches submitted to accumulate stats from 4480 documents (362576 virtual) 2018-03-28 00:36:06,804 : INFO : 71 batches submitted to accumulate stats from 4544 documents (367317 virtual) 2018-03-28 00:36:06,859 : INFO : 72 batches submitted to accumulate stats from 4608 documents (372593 virtual) 2018-03-28 00:36:06,973 : INFO : 73 batches submitted to accumulate stats from 4672 documents (377717 virtual) 2018-03-28 00:36:06,982 : INFO : 74 batches submitted to accumulate stats from 4736 documents (383450 virtual) 2018-03-28 00:36:07,065 : INFO : 75 batches submitted to accumulate stats from 4800 documents (388532 virtual) 2018-03-28 00:36:07,177 : INFO : 76 batches submitted to accumulate stats from 4864 documents (393133 virtual) 2018-03-28 00:36:07,197 : INFO : 77 batches submitted to accumulate stats from 4928 documents (398349 virtual) 2018-03-28 00:36:07,249 : INFO : 78 batches submitted to accumulate stats from 4992 documents (403438 virtual) 2018-03-28 00:36:07,372 : INFO : 79 batches submitted to accumulate stats from 5056 documents (408497 virtual) 2018-03-28 00:36:07,412 : INFO : 80 batches submitted to accumulate stats from 5120 documents (413883 virtual) 2018-03-28 00:36:07,448 : INFO : 81 batches submitted to accumulate stats from 5184 documents (419375 virtual) 2018-03-28 00:36:07,556 : INFO : 82 batches submitted to accumulate stats from 5248 documents (424667 virtual) 2018-03-28 00:36:07,589 : INFO : 83 batches submitted to accumulate stats from 5312 documents (429889 virtual) 2018-03-28 00:36:07,654 : INFO : 84 batches submitted to accumulate stats from 5376 documents (435838 virtual) 2018-03-28 00:36:07,745 : INFO : 85 batches submitted to accumulate stats from 5440 documents (441360 virtual) 2018-03-28 00:36:07,816 : INFO : 86 batches submitted to accumulate stats from 5504 documents (447116 virtual) 2018-03-28 00:36:07,920 : INFO : 87 batches submitted to accumulate stats from 5568 documents (451645 virtual) 2018-03-28 00:36:07,927 : INFO : 88 batches submitted to accumulate stats from 5632 documents (457033 virtual) 2018-03-28 00:36:08,008 : INFO : 89 batches submitted to accumulate stats from 5696 documents (462287 virtual) 2018-03-28 00:36:08,117 : INFO : 90 batches submitted to accumulate stats from 5760 documents (467656 virtual) 2018-03-28 00:36:08,187 : INFO : 91 batches submitted to accumulate stats from 5824 documents (472972 virtual) 2018-03-28 00:36:08,215 : INFO : 92 batches submitted to accumulate stats from 5888 documents (478670 virtual) 2018-03-28 00:36:08,287 : INFO : 93 batches submitted to accumulate stats from 5952 documents (483600 virtual) 2018-03-28 00:36:08,388 : INFO : 94 batches submitted to accumulate stats from 6016 documents (489170 virtual) 2018-03-28 00:36:08,410 : INFO : 95 batches submitted to accumulate stats from 6080 documents (494885 virtual) 2018-03-28 00:36:08,526 : INFO : 96 batches submitted to accumulate stats from 6144 documents (500078 virtual) 2018-03-28 00:36:08,593 : INFO : 97 batches submitted to accumulate stats from 6208 documents (505384 virtual) 2018-03-28 00:36:08,610 : INFO : 98 batches submitted to accumulate stats from 6272 documents (510422 virtual) 2018-03-28 00:36:08,747 : INFO : 99 batches submitted to accumulate stats from 6336 documents (515440 virtual) 2018-03-28 00:36:08,791 : INFO : 100 batches submitted to accumulate stats from 6400 documents (520316 virtual) 2018-03-28 00:36:08,825 : INFO : 101 batches submitted to accumulate stats from 6464 documents (525437 virtual) 2018-03-28 00:36:08,969 : INFO : 102 batches submitted to accumulate stats from 6528 documents (530499 virtual) 2018-03-28 00:36:08,976 : INFO : 103 batches submitted to accumulate stats from 6592 documents (535463 virtual) 2018-03-28 00:36:09,037 : INFO : 104 batches submitted to accumulate stats from 6656 documents (540758 virtual) 2018-03-28 00:36:09,170 : INFO : 105 batches submitted to accumulate stats from 6720 documents (546012 virtual) 2018-03-28 00:36:09,177 : INFO : 106 batches submitted to accumulate stats from 6784 documents (550852 virtual) 2018-03-28 00:36:09,211 : INFO : 107 batches submitted to accumulate stats from 6848 documents (556235 virtual) 2018-03-28 00:36:09,364 : INFO : 108 batches submitted to accumulate stats from 6912 documents (561393 virtual) 2018-03-28 00:36:09,373 : INFO : 109 batches submitted to accumulate stats from 6976 documents (566997 virtual) 2018-03-28 00:36:09,408 : INFO : 110 batches submitted to accumulate stats from 7040 documents (571906 virtual) 2018-03-28 00:36:09,552 : INFO : 111 batches submitted to accumulate stats from 7104 documents (576643 virtual) 2018-03-28 00:36:09,586 : INFO : 112 batches submitted to accumulate stats from 7168 documents (581432 virtual) 2018-03-28 00:36:09,613 : INFO : 113 batches submitted to accumulate stats from 7232 documents (586908 virtual) 2018-03-28 00:36:09,766 : INFO : 114 batches submitted to accumulate stats from 7296 documents (591452 virtual) 2018-03-28 00:36:09,835 : INFO : 115 batches submitted to accumulate stats from 7360 documents (596898 virtual) 2018-03-28 00:36:09,871 : INFO : 116 batches submitted to accumulate stats from 7424 documents (602443 virtual) 2018-03-28 00:36:09,979 : INFO : 117 batches submitted to accumulate stats from 7488 documents (607306 virtual) 2018-03-28 00:36:10,017 : INFO : 118 batches submitted to accumulate stats from 7552 documents (612546 virtual) 2018-03-28 00:36:10,092 : INFO : 119 batches submitted to accumulate stats from 7616 documents (617634 virtual) 2018-03-28 00:36:10,153 : INFO : 120 batches submitted to accumulate stats from 7680 documents (622816 virtual) 2018-03-28 00:36:10,241 : INFO : 121 batches submitted to accumulate stats from 7744 documents (627780 virtual) 2018-03-28 00:36:10,312 : INFO : 122 batches submitted to accumulate stats from 7808 documents (633874 virtual) 2018-03-28 00:36:10,319 : INFO : 123 batches submitted to accumulate stats from 7872 documents (638920 virtual) 2018-03-28 00:36:10,428 : INFO : 124 batches submitted to accumulate stats from 7936 documents (643953 virtual) 2018-03-28 00:36:10,516 : INFO : 125 batches submitted to accumulate stats from 8000 documents (647636 virtual) 2018-03-28 00:36:10,748 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:10,822 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:10,752 : INFO : accumulator serialized 2018-03-28 00:36:10,871 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:10,829 : INFO : accumulator serialized 2018-03-28 00:36:10,877 : INFO : accumulator serialized 2018-03-28 00:36:10,988 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:36:11,154 : INFO : accumulated word occurrence stats for 648298 virtual documents 2018-03-28 00:36:12,024 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_35 2018-03-28 00:36:12,026 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_35.expElogbeta.npy with mmap=None 2018-03-28 00:36:12,029 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:36:12,030 : INFO : setting ignored attribute state to None 2018-03-28 00:36:12,030 : INFO : setting ignored attribute id2word to None 2018-03-28 00:36:12,031 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_35 2018-03-28 00:36:12,031 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_35.state 2018-03-28 00:36:12,048 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_35.state 2018-03-28 00:36:12,068 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:36:12,115 : INFO : 1 batches submitted to accumulate stats from 64 documents (4982 virtual) 2018-03-28 00:36:12,122 : INFO : 2 batches submitted to accumulate stats from 128 documents (10157 virtual) 2018-03-28 00:36:12,129 : INFO : 3 batches submitted to accumulate stats from 192 documents (15057 virtual) 2018-03-28 00:36:12,137 : INFO : 4 batches submitted to accumulate stats from 256 documents (19941 virtual) 2018-03-28 00:36:12,143 : INFO : 5 batches submitted to accumulate stats from 320 documents (25048 virtual) 2018-03-28 00:36:12,150 : INFO : 6 batches submitted to accumulate stats from 384 documents (30581 virtual) 2018-03-28 00:36:12,356 : INFO : 7 batches submitted to accumulate stats from 448 documents (35845 virtual) 2018-03-28 00:36:12,393 : INFO : 8 batches submitted to accumulate stats from 512 documents (40777 virtual) 2018-03-28 00:36:12,408 : INFO : 9 batches submitted to accumulate stats from 576 documents (45638 virtual) 2018-03-28 00:36:12,564 : INFO : 10 batches submitted to accumulate stats from 640 documents (51118 virtual) 2018-03-28 00:36:12,613 : INFO : 11 batches submitted to accumulate stats from 704 documents (56333 virtual) 2018-03-28 00:36:12,626 : INFO : 12 batches submitted to accumulate stats from 768 documents (60983 virtual) 2018-03-28 00:36:12,789 : INFO : 13 batches submitted to accumulate stats from 832 documents (66662 virtual) 2018-03-28 00:36:12,804 : INFO : 14 batches submitted to accumulate stats from 896 documents (72096 virtual) 2018-03-28 00:36:12,850 : INFO : 15 batches submitted to accumulate stats from 960 documents (77271 virtual) 2018-03-28 00:36:13,008 : INFO : 16 batches submitted to accumulate stats from 1024 documents (83303 virtual) 2018-03-28 00:36:13,019 : INFO : 17 batches submitted to accumulate stats from 1088 documents (88427 virtual) 2018-03-28 00:36:13,045 : INFO : 18 batches submitted to accumulate stats from 1152 documents (93247 virtual) 2018-03-28 00:36:13,212 : INFO : 19 batches submitted to accumulate stats from 1216 documents (98043 virtual) 2018-03-28 00:36:13,260 : INFO : 20 batches submitted to accumulate stats from 1280 documents (103776 virtual) 2018-03-28 00:36:13,266 : INFO : 21 batches submitted to accumulate stats from 1344 documents (109033 virtual) 2018-03-28 00:36:13,446 : INFO : 22 batches submitted to accumulate stats from 1408 documents (114396 virtual) 2018-03-28 00:36:13,458 : INFO : 23 batches submitted to accumulate stats from 1472 documents (120092 virtual) 2018-03-28 00:36:13,477 : INFO : 24 batches submitted to accumulate stats from 1536 documents (125140 virtual) 2018-03-28 00:36:13,665 : INFO : 25 batches submitted to accumulate stats from 1600 documents (130666 virtual) 2018-03-28 00:36:13,671 : INFO : 26 batches submitted to accumulate stats from 1664 documents (135545 virtual) 2018-03-28 00:36:13,686 : INFO : 27 batches submitted to accumulate stats from 1728 documents (140960 virtual) 2018-03-28 00:36:13,884 : INFO : 28 batches submitted to accumulate stats from 1792 documents (146130 virtual) 2018-03-28 00:36:13,891 : INFO : 29 batches submitted to accumulate stats from 1856 documents (151717 virtual) 2018-03-28 00:36:13,898 : INFO : 30 batches submitted to accumulate stats from 1920 documents (157307 virtual) 2018-03-28 00:36:14,062 : INFO : 31 batches submitted to accumulate stats from 1984 documents (162264 virtual) 2018-03-28 00:36:14,115 : INFO : 32 batches submitted to accumulate stats from 2048 documents (167215 virtual) 2018-03-28 00:36:14,147 : INFO : 33 batches submitted to accumulate stats from 2112 documents (172021 virtual) 2018-03-28 00:36:14,276 : INFO : 34 batches submitted to accumulate stats from 2176 documents (177510 virtual) 2018-03-28 00:36:14,347 : INFO : 35 batches submitted to accumulate stats from 2240 documents (183041 virtual) 2018-03-28 00:36:14,373 : INFO : 36 batches submitted to accumulate stats from 2304 documents (188101 virtual) 2018-03-28 00:36:14,483 : INFO : 37 batches submitted to accumulate stats from 2368 documents (193768 virtual) 2018-03-28 00:36:14,572 : INFO : 38 batches submitted to accumulate stats from 2432 documents (200069 virtual) 2018-03-28 00:36:14,578 : INFO : 39 batches submitted to accumulate stats from 2496 documents (205101 virtual) 2018-03-28 00:36:14,712 : INFO : 40 batches submitted to accumulate stats from 2560 documents (210117 virtual) 2018-03-28 00:36:14,777 : INFO : 41 batches submitted to accumulate stats from 2624 documents (214878 virtual) 2018-03-28 00:36:14,799 : INFO : 42 batches submitted to accumulate stats from 2688 documents (220029 virtual) 2018-03-28 00:36:14,936 : INFO : 43 batches submitted to accumulate stats from 2752 documents (224625 virtual) 2018-03-28 00:36:15,008 : INFO : 44 batches submitted to accumulate stats from 2816 documents (229171 virtual) 2018-03-28 00:36:15,030 : INFO : 45 batches submitted to accumulate stats from 2880 documents (234770 virtual) 2018-03-28 00:36:15,177 : INFO : 46 batches submitted to accumulate stats from 2944 documents (239643 virtual) 2018-03-28 00:36:15,186 : INFO : 47 batches submitted to accumulate stats from 3008 documents (244507 virtual) 2018-03-28 00:36:15,249 : INFO : 48 batches submitted to accumulate stats from 3072 documents (249507 virtual) 2018-03-28 00:36:15,348 : INFO : 49 batches submitted to accumulate stats from 3136 documents (254338 virtual) 2018-03-28 00:36:15,408 : INFO : 50 batches submitted to accumulate stats from 3200 documents (259842 virtual) 2018-03-28 00:36:15,479 : INFO : 51 batches submitted to accumulate stats from 3264 documents (264839 virtual) 2018-03-28 00:36:15,557 : INFO : 52 batches submitted to accumulate stats from 3328 documents (270105 virtual) 2018-03-28 00:36:15,610 : INFO : 53 batches submitted to accumulate stats from 3392 documents (275383 virtual) 2018-03-28 00:36:15,708 : INFO : 54 batches submitted to accumulate stats from 3456 documents (280287 virtual) 2018-03-28 00:36:15,754 : INFO : 55 batches submitted to accumulate stats from 3520 documents (285341 virtual) 2018-03-28 00:36:15,833 : INFO : 56 batches submitted to accumulate stats from 3584 documents (290319 virtual) 2018-03-28 00:36:15,889 : INFO : 57 batches submitted to accumulate stats from 3648 documents (295502 virtual) 2018-03-28 00:36:16,007 : INFO : 58 batches submitted to accumulate stats from 3712 documents (300916 virtual) 2018-03-28 00:36:16,041 : INFO : 59 batches submitted to accumulate stats from 3776 documents (306009 virtual) 2018-03-28 00:36:16,090 : INFO : 60 batches submitted to accumulate stats from 3840 documents (311303 virtual) 2018-03-28 00:36:16,219 : INFO : 61 batches submitted to accumulate stats from 3904 documents (316424 virtual) 2018-03-28 00:36:16,251 : INFO : 62 batches submitted to accumulate stats from 3968 documents (321297 virtual) 2018-03-28 00:36:16,301 : INFO : 63 batches submitted to accumulate stats from 4032 documents (325989 virtual) 2018-03-28 00:36:16,432 : INFO : 64 batches submitted to accumulate stats from 4096 documents (331472 virtual) 2018-03-28 00:36:16,479 : INFO : 65 batches submitted to accumulate stats from 4160 documents (336773 virtual) 2018-03-28 00:36:16,535 : INFO : 66 batches submitted to accumulate stats from 4224 documents (341920 virtual) 2018-03-28 00:36:16,657 : INFO : 67 batches submitted to accumulate stats from 4288 documents (347006 virtual) 2018-03-28 00:36:16,693 : INFO : 68 batches submitted to accumulate stats from 4352 documents (351876 virtual) 2018-03-28 00:36:16,719 : INFO : 69 batches submitted to accumulate stats from 4416 documents (356999 virtual) 2018-03-28 00:36:16,893 : INFO : 70 batches submitted to accumulate stats from 4480 documents (362347 virtual) 2018-03-28 00:36:16,921 : INFO : 71 batches submitted to accumulate stats from 4544 documents (367042 virtual) 2018-03-28 00:36:16,928 : INFO : 72 batches submitted to accumulate stats from 4608 documents (372395 virtual) 2018-03-28 00:36:17,095 : INFO : 73 batches submitted to accumulate stats from 4672 documents (377514 virtual) 2018-03-28 00:36:17,102 : INFO : 74 batches submitted to accumulate stats from 4736 documents (383113 virtual) 2018-03-28 00:36:17,180 : INFO : 75 batches submitted to accumulate stats from 4800 documents (388117 virtual) 2018-03-28 00:36:17,285 : INFO : 76 batches submitted to accumulate stats from 4864 documents (392844 virtual) 2018-03-28 00:36:17,364 : INFO : 77 batches submitted to accumulate stats from 4928 documents (398082 virtual) 2018-03-28 00:36:17,439 : INFO : 78 batches submitted to accumulate stats from 4992 documents (403199 virtual) 2018-03-28 00:36:17,503 : INFO : 79 batches submitted to accumulate stats from 5056 documents (408222 virtual) 2018-03-28 00:36:17,604 : INFO : 80 batches submitted to accumulate stats from 5120 documents (413552 virtual) 2018-03-28 00:36:17,646 : INFO : 81 batches submitted to accumulate stats from 5184 documents (419000 virtual) 2018-03-28 00:36:17,721 : INFO : 82 batches submitted to accumulate stats from 5248 documents (424334 virtual) 2018-03-28 00:36:17,810 : INFO : 83 batches submitted to accumulate stats from 5312 documents (429623 virtual) 2018-03-28 00:36:17,857 : INFO : 84 batches submitted to accumulate stats from 5376 documents (435549 virtual) 2018-03-28 00:36:17,934 : INFO : 85 batches submitted to accumulate stats from 5440 documents (441018 virtual) 2018-03-28 00:36:18,026 : INFO : 86 batches submitted to accumulate stats from 5504 documents (446736 virtual) 2018-03-28 00:36:18,097 : INFO : 87 batches submitted to accumulate stats from 5568 documents (451230 virtual) 2018-03-28 00:36:18,173 : INFO : 88 batches submitted to accumulate stats from 5632 documents (456628 virtual) 2018-03-28 00:36:18,233 : INFO : 89 batches submitted to accumulate stats from 5696 documents (461820 virtual) 2018-03-28 00:36:18,355 : INFO : 90 batches submitted to accumulate stats from 5760 documents (467303 virtual) 2018-03-28 00:36:18,392 : INFO : 91 batches submitted to accumulate stats from 5824 documents (472558 virtual) 2018-03-28 00:36:18,482 : INFO : 92 batches submitted to accumulate stats from 5888 documents (478313 virtual) 2018-03-28 00:36:18,585 : INFO : 93 batches submitted to accumulate stats from 5952 documents (483414 virtual) 2018-03-28 00:36:18,691 : INFO : 94 batches submitted to accumulate stats from 6016 documents (488906 virtual) 2018-03-28 00:36:18,785 : INFO : 95 batches submitted to accumulate stats from 6080 documents (494363 virtual) 2018-03-28 00:36:18,884 : INFO : 96 batches submitted to accumulate stats from 6144 documents (499815 virtual) 2018-03-28 00:36:18,987 : INFO : 97 batches submitted to accumulate stats from 6208 documents (505068 virtual) 2018-03-28 00:36:19,058 : INFO : 98 batches submitted to accumulate stats from 6272 documents (510055 virtual) 2018-03-28 00:36:19,197 : INFO : 99 batches submitted to accumulate stats from 6336 documents (515019 virtual) 2018-03-28 00:36:19,262 : INFO : 100 batches submitted to accumulate stats from 6400 documents (520003 virtual) 2018-03-28 00:36:19,317 : INFO : 101 batches submitted to accumulate stats from 6464 documents (525265 virtual) 2018-03-28 00:36:19,424 : INFO : 102 batches submitted to accumulate stats from 6528 documents (530099 virtual) 2018-03-28 00:36:19,497 : INFO : 103 batches submitted to accumulate stats from 6592 documents (535097 virtual) 2018-03-28 00:36:19,515 : INFO : 104 batches submitted to accumulate stats from 6656 documents (540175 virtual) 2018-03-28 00:36:19,639 : INFO : 105 batches submitted to accumulate stats from 6720 documents (545797 virtual) 2018-03-28 00:36:19,701 : INFO : 106 batches submitted to accumulate stats from 6784 documents (550508 virtual) 2018-03-28 00:36:19,716 : INFO : 107 batches submitted to accumulate stats from 6848 documents (555898 virtual) 2018-03-28 00:36:19,850 : INFO : 108 batches submitted to accumulate stats from 6912 documents (560666 virtual) 2018-03-28 00:36:19,896 : INFO : 109 batches submitted to accumulate stats from 6976 documents (566668 virtual) 2018-03-28 00:36:19,943 : INFO : 110 batches submitted to accumulate stats from 7040 documents (571469 virtual) 2018-03-28 00:36:20,061 : INFO : 111 batches submitted to accumulate stats from 7104 documents (576162 virtual) 2018-03-28 00:36:20,094 : INFO : 112 batches submitted to accumulate stats from 7168 documents (580837 virtual) 2018-03-28 00:36:20,228 : INFO : 113 batches submitted to accumulate stats from 7232 documents (586464 virtual) 2018-03-28 00:36:20,234 : INFO : 114 batches submitted to accumulate stats from 7296 documents (590878 virtual) 2018-03-28 00:36:20,346 : INFO : 115 batches submitted to accumulate stats from 7360 documents (596510 virtual) 2018-03-28 00:36:20,474 : INFO : 116 batches submitted to accumulate stats from 7424 documents (602165 virtual) 2018-03-28 00:36:20,483 : INFO : 117 batches submitted to accumulate stats from 7488 documents (607054 virtual) 2018-03-28 00:36:20,600 : INFO : 118 batches submitted to accumulate stats from 7552 documents (612133 virtual) 2018-03-28 00:36:20,733 : INFO : 119 batches submitted to accumulate stats from 7616 documents (617179 virtual) 2018-03-28 00:36:20,782 : INFO : 120 batches submitted to accumulate stats from 7680 documents (622297 virtual) 2018-03-28 00:36:20,859 : INFO : 121 batches submitted to accumulate stats from 7744 documents (627275 virtual) 2018-03-28 00:36:20,998 : INFO : 122 batches submitted to accumulate stats from 7808 documents (633329 virtual) 2018-03-28 00:36:21,012 : INFO : 123 batches submitted to accumulate stats from 7872 documents (638391 virtual) 2018-03-28 00:36:21,074 : INFO : 124 batches submitted to accumulate stats from 7936 documents (643487 virtual) 2018-03-28 00:36:21,190 : INFO : 125 batches submitted to accumulate stats from 8000 documents (647634 virtual) 2018-03-28 00:36:21,447 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:21,456 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:21,452 : INFO : accumulator serialized 2018-03-28 00:36:21,463 : INFO : accumulator serialized 2018-03-28 00:36:21,604 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:21,609 : INFO : accumulator serialized 2018-03-28 00:36:21,728 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:36:21,924 : INFO : accumulated word occurrence stats for 648305 virtual documents 2018-03-28 00:36:22,900 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_40 2018-03-28 00:36:22,902 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-28 00:36:22,905 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:36:22,906 : INFO : setting ignored attribute state to None 2018-03-28 00:36:22,906 : INFO : setting ignored attribute id2word to None 2018-03-28 00:36:22,907 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_40 2018-03-28 00:36:22,908 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_40.state 2018-03-28 00:36:22,926 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_40.state 2018-03-28 00:36:22,949 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:36:22,996 : INFO : 1 batches submitted to accumulate stats from 64 documents (4982 virtual) 2018-03-28 00:36:23,004 : INFO : 2 batches submitted to accumulate stats from 128 documents (10157 virtual) 2018-03-28 00:36:23,010 : INFO : 3 batches submitted to accumulate stats from 192 documents (15057 virtual) 2018-03-28 00:36:23,018 : INFO : 4 batches submitted to accumulate stats from 256 documents (19941 virtual) 2018-03-28 00:36:23,025 : INFO : 5 batches submitted to accumulate stats from 320 documents (25048 virtual) 2018-03-28 00:36:23,032 : INFO : 6 batches submitted to accumulate stats from 384 documents (30581 virtual) 2018-03-28 00:36:23,233 : INFO : 7 batches submitted to accumulate stats from 448 documents (35845 virtual) 2018-03-28 00:36:23,240 : INFO : 8 batches submitted to accumulate stats from 512 documents (40777 virtual) 2018-03-28 00:36:23,247 : INFO : 9 batches submitted to accumulate stats from 576 documents (45638 virtual) 2018-03-28 00:36:23,418 : INFO : 10 batches submitted to accumulate stats from 640 documents (51118 virtual) 2018-03-28 00:36:23,485 : INFO : 11 batches submitted to accumulate stats from 704 documents (56333 virtual) 2018-03-28 00:36:23,517 : INFO : 12 batches submitted to accumulate stats from 768 documents (61060 virtual) 2018-03-28 00:36:23,674 : INFO : 13 batches submitted to accumulate stats from 832 documents (66653 virtual) 2018-03-28 00:36:23,714 : INFO : 14 batches submitted to accumulate stats from 896 documents (72087 virtual) 2018-03-28 00:36:23,721 : INFO : 15 batches submitted to accumulate stats from 960 documents (77262 virtual) 2018-03-28 00:36:23,912 : INFO : 16 batches submitted to accumulate stats from 1024 documents (83294 virtual) 2018-03-28 00:36:23,970 : INFO : 17 batches submitted to accumulate stats from 1088 documents (88418 virtual) 2018-03-28 00:36:23,976 : INFO : 18 batches submitted to accumulate stats from 1152 documents (93238 virtual) 2018-03-28 00:36:24,203 : INFO : 19 batches submitted to accumulate stats from 1216 documents (98034 virtual) 2018-03-28 00:36:24,222 : INFO : 20 batches submitted to accumulate stats from 1280 documents (103767 virtual) 2018-03-28 00:36:24,278 : INFO : 21 batches submitted to accumulate stats from 1344 documents (109024 virtual) 2018-03-28 00:36:24,456 : INFO : 22 batches submitted to accumulate stats from 1408 documents (114387 virtual) 2018-03-28 00:36:24,489 : INFO : 23 batches submitted to accumulate stats from 1472 documents (120083 virtual) 2018-03-28 00:36:24,523 : INFO : 24 batches submitted to accumulate stats from 1536 documents (125131 virtual) 2018-03-28 00:36:24,671 : INFO : 25 batches submitted to accumulate stats from 1600 documents (130657 virtual) 2018-03-28 00:36:24,731 : INFO : 26 batches submitted to accumulate stats from 1664 documents (135536 virtual) 2018-03-28 00:36:24,775 : INFO : 27 batches submitted to accumulate stats from 1728 documents (140951 virtual) 2018-03-28 00:36:24,888 : INFO : 28 batches submitted to accumulate stats from 1792 documents (146121 virtual) 2018-03-28 00:36:24,964 : INFO : 29 batches submitted to accumulate stats from 1856 documents (151708 virtual) 2018-03-28 00:36:25,014 : INFO : 30 batches submitted to accumulate stats from 1920 documents (157298 virtual) 2018-03-28 00:36:25,129 : INFO : 31 batches submitted to accumulate stats from 1984 documents (162255 virtual) 2018-03-28 00:36:25,213 : INFO : 32 batches submitted to accumulate stats from 2048 documents (167206 virtual) 2018-03-28 00:36:25,219 : INFO : 33 batches submitted to accumulate stats from 2112 documents (172012 virtual) 2018-03-28 00:36:25,352 : INFO : 34 batches submitted to accumulate stats from 2176 documents (177432 virtual) 2018-03-28 00:36:25,462 : INFO : 35 batches submitted to accumulate stats from 2240 documents (182931 virtual) 2018-03-28 00:36:25,494 : INFO : 36 batches submitted to accumulate stats from 2304 documents (188013 virtual) 2018-03-28 00:36:25,553 : INFO : 37 batches submitted to accumulate stats from 2368 documents (193729 virtual) 2018-03-28 00:36:25,669 : INFO : 38 batches submitted to accumulate stats from 2432 documents (199970 virtual) 2018-03-28 00:36:25,710 : INFO : 39 batches submitted to accumulate stats from 2496 documents (205019 virtual) 2018-03-28 00:36:25,796 : INFO : 40 batches submitted to accumulate stats from 2560 documents (210023 virtual) 2018-03-28 00:36:25,912 : INFO : 41 batches submitted to accumulate stats from 2624 documents (214806 virtual) 2018-03-28 00:36:25,921 : INFO : 42 batches submitted to accumulate stats from 2688 documents (220000 virtual) 2018-03-28 00:36:26,069 : INFO : 43 batches submitted to accumulate stats from 2752 documents (224577 virtual) 2018-03-28 00:36:26,092 : INFO : 44 batches submitted to accumulate stats from 2816 documents (229091 virtual) 2018-03-28 00:36:26,203 : INFO : 45 batches submitted to accumulate stats from 2880 documents (234690 virtual) 2018-03-28 00:36:26,246 : INFO : 46 batches submitted to accumulate stats from 2944 documents (239521 virtual) 2018-03-28 00:36:26,317 : INFO : 47 batches submitted to accumulate stats from 3008 documents (244483 virtual) 2018-03-28 00:36:26,407 : INFO : 48 batches submitted to accumulate stats from 3072 documents (249446 virtual) 2018-03-28 00:36:26,477 : INFO : 49 batches submitted to accumulate stats from 3136 documents (254308 virtual) 2018-03-28 00:36:26,524 : INFO : 50 batches submitted to accumulate stats from 3200 documents (259741 virtual) 2018-03-28 00:36:26,629 : INFO : 51 batches submitted to accumulate stats from 3264 documents (264836 virtual) 2018-03-28 00:36:26,690 : INFO : 52 batches submitted to accumulate stats from 3328 documents (269983 virtual) 2018-03-28 00:36:26,761 : INFO : 53 batches submitted to accumulate stats from 3392 documents (275276 virtual) 2018-03-28 00:36:26,846 : INFO : 54 batches submitted to accumulate stats from 3456 documents (280281 virtual) 2018-03-28 00:36:26,907 : INFO : 55 batches submitted to accumulate stats from 3520 documents (285264 virtual) 2018-03-28 00:36:26,992 : INFO : 56 batches submitted to accumulate stats from 3584 documents (290232 virtual) 2018-03-28 00:36:27,060 : INFO : 57 batches submitted to accumulate stats from 3648 documents (295448 virtual) 2018-03-28 00:36:27,129 : INFO : 58 batches submitted to accumulate stats from 3712 documents (300819 virtual) 2018-03-28 00:36:27,227 : INFO : 59 batches submitted to accumulate stats from 3776 documents (305975 virtual) 2018-03-28 00:36:27,312 : INFO : 60 batches submitted to accumulate stats from 3840 documents (310984 virtual) 2018-03-28 00:36:27,367 : INFO : 61 batches submitted to accumulate stats from 3904 documents (316238 virtual) 2018-03-28 00:36:27,497 : INFO : 62 batches submitted to accumulate stats from 3968 documents (320871 virtual) 2018-03-28 00:36:27,534 : INFO : 63 batches submitted to accumulate stats from 4032 documents (325840 virtual) 2018-03-28 00:36:27,584 : INFO : 64 batches submitted to accumulate stats from 4096 documents (331190 virtual) 2018-03-28 00:36:27,754 : INFO : 65 batches submitted to accumulate stats from 4160 documents (336204 virtual) 2018-03-28 00:36:27,767 : INFO : 66 batches submitted to accumulate stats from 4224 documents (341653 virtual) 2018-03-28 00:36:27,794 : INFO : 67 batches submitted to accumulate stats from 4288 documents (346672 virtual) 2018-03-28 00:36:27,979 : INFO : 68 batches submitted to accumulate stats from 4352 documents (351560 virtual) 2018-03-28 00:36:27,995 : INFO : 69 batches submitted to accumulate stats from 4416 documents (356754 virtual) 2018-03-28 00:36:28,001 : INFO : 70 batches submitted to accumulate stats from 4480 documents (362007 virtual) 2018-03-28 00:36:28,192 : INFO : 71 batches submitted to accumulate stats from 4544 documents (366687 virtual) 2018-03-28 00:36:28,209 : INFO : 72 batches submitted to accumulate stats from 4608 documents (372025 virtual) 2018-03-28 00:36:28,239 : INFO : 73 batches submitted to accumulate stats from 4672 documents (377004 virtual) 2018-03-28 00:36:28,395 : INFO : 74 batches submitted to accumulate stats from 4736 documents (382675 virtual) 2018-03-28 00:36:28,450 : INFO : 75 batches submitted to accumulate stats from 4800 documents (387833 virtual) 2018-03-28 00:36:28,457 : INFO : 76 batches submitted to accumulate stats from 4864 documents (392595 virtual) 2018-03-28 00:36:28,625 : INFO : 77 batches submitted to accumulate stats from 4928 documents (397610 virtual) 2018-03-28 00:36:28,668 : INFO : 78 batches submitted to accumulate stats from 4992 documents (402897 virtual) 2018-03-28 00:36:28,685 : INFO : 79 batches submitted to accumulate stats from 5056 documents (407892 virtual) 2018-03-28 00:36:28,849 : INFO : 80 batches submitted to accumulate stats from 5120 documents (413289 virtual) 2018-03-28 00:36:28,897 : INFO : 81 batches submitted to accumulate stats from 5184 documents (418818 virtual) 2018-03-28 00:36:28,907 : INFO : 82 batches submitted to accumulate stats from 5248 documents (423833 virtual) 2018-03-28 00:36:29,048 : INFO : 83 batches submitted to accumulate stats from 5312 documents (429231 virtual) 2018-03-28 00:36:29,116 : INFO : 84 batches submitted to accumulate stats from 5376 documents (434923 virtual) 2018-03-28 00:36:29,164 : INFO : 85 batches submitted to accumulate stats from 5440 documents (440428 virtual) 2018-03-28 00:36:29,272 : INFO : 86 batches submitted to accumulate stats from 5504 documents (446295 virtual) 2018-03-28 00:36:29,332 : INFO : 87 batches submitted to accumulate stats from 5568 documents (450695 virtual) 2018-03-28 00:36:29,401 : INFO : 88 batches submitted to accumulate stats from 5632 documents (456089 virtual) 2018-03-28 00:36:29,483 : INFO : 89 batches submitted to accumulate stats from 5696 documents (461264 virtual) 2018-03-28 00:36:29,626 : INFO : 90 batches submitted to accumulate stats from 5760 documents (466660 virtual) 2018-03-28 00:36:29,637 : INFO : 91 batches submitted to accumulate stats from 5824 documents (472113 virtual) 2018-03-28 00:36:29,713 : INFO : 92 batches submitted to accumulate stats from 5888 documents (477564 virtual) 2018-03-28 00:36:29,824 : INFO : 93 batches submitted to accumulate stats from 5952 documents (482758 virtual) 2018-03-28 00:36:29,868 : INFO : 94 batches submitted to accumulate stats from 6016 documents (488334 virtual) 2018-03-28 00:36:29,925 : INFO : 95 batches submitted to accumulate stats from 6080 documents (493868 virtual) 2018-03-28 00:36:30,060 : INFO : 96 batches submitted to accumulate stats from 6144 documents (499242 virtual) 2018-03-28 00:36:30,084 : INFO : 97 batches submitted to accumulate stats from 6208 documents (504636 virtual) 2018-03-28 00:36:30,169 : INFO : 98 batches submitted to accumulate stats from 6272 documents (509567 virtual) 2018-03-28 00:36:30,263 : INFO : 99 batches submitted to accumulate stats from 6336 documents (514572 virtual) 2018-03-28 00:36:30,342 : INFO : 100 batches submitted to accumulate stats from 6400 documents (519486 virtual) 2018-03-28 00:36:30,459 : INFO : 101 batches submitted to accumulate stats from 6464 documents (524871 virtual) 2018-03-28 00:36:30,467 : INFO : 102 batches submitted to accumulate stats from 6528 documents (529588 virtual) 2018-03-28 00:36:30,605 : INFO : 103 batches submitted to accumulate stats from 6592 documents (534520 virtual) 2018-03-28 00:36:30,651 : INFO : 104 batches submitted to accumulate stats from 6656 documents (539677 virtual) 2018-03-28 00:36:30,673 : INFO : 105 batches submitted to accumulate stats from 6720 documents (545088 virtual) 2018-03-28 00:36:30,813 : INFO : 106 batches submitted to accumulate stats from 6784 documents (549949 virtual) 2018-03-28 00:36:30,881 : INFO : 107 batches submitted to accumulate stats from 6848 documents (555265 virtual) 2018-03-28 00:36:30,887 : INFO : 108 batches submitted to accumulate stats from 6912 documents (559967 virtual) 2018-03-28 00:36:31,056 : INFO : 109 batches submitted to accumulate stats from 6976 documents (565982 virtual) 2018-03-28 00:36:31,091 : INFO : 110 batches submitted to accumulate stats from 7040 documents (570710 virtual) 2018-03-28 00:36:31,130 : INFO : 111 batches submitted to accumulate stats from 7104 documents (575620 virtual) 2018-03-28 00:36:31,248 : INFO : 112 batches submitted to accumulate stats from 7168 documents (580261 virtual) 2018-03-28 00:36:31,334 : INFO : 113 batches submitted to accumulate stats from 7232 documents (585711 virtual) 2018-03-28 00:36:31,351 : INFO : 114 batches submitted to accumulate stats from 7296 documents (590386 virtual) 2018-03-28 00:36:31,528 : INFO : 115 batches submitted to accumulate stats from 7360 documents (595608 virtual) 2018-03-28 00:36:31,541 : INFO : 116 batches submitted to accumulate stats from 7424 documents (601103 virtual) 2018-03-28 00:36:31,547 : INFO : 117 batches submitted to accumulate stats from 7488 documents (606539 virtual) 2018-03-28 00:36:31,713 : INFO : 118 batches submitted to accumulate stats from 7552 documents (611440 virtual) 2018-03-28 00:36:31,770 : INFO : 119 batches submitted to accumulate stats from 7616 documents (616536 virtual) 2018-03-28 00:36:31,785 : INFO : 120 batches submitted to accumulate stats from 7680 documents (621619 virtual) 2018-03-28 00:36:31,935 : INFO : 121 batches submitted to accumulate stats from 7744 documents (626653 virtual) 2018-03-28 00:36:32,012 : INFO : 122 batches submitted to accumulate stats from 7808 documents (632365 virtual) 2018-03-28 00:36:32,026 : INFO : 123 batches submitted to accumulate stats from 7872 documents (637405 virtual) 2018-03-28 00:36:32,165 : INFO : 124 batches submitted to accumulate stats from 7936 documents (642818 virtual) 2018-03-28 00:36:32,225 : INFO : 125 batches submitted to accumulate stats from 8000 documents (647614 virtual) 2018-03-28 00:36:32,484 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:32,569 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:32,489 : INFO : accumulator serialized 2018-03-28 00:36:32,700 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:32,576 : INFO : accumulator serialized 2018-03-28 00:36:32,707 : INFO : accumulator serialized 2018-03-28 00:36:32,850 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:36:33,095 : INFO : accumulated word occurrence stats for 648317 virtual documents 2018-03-28 00:36:34,229 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_50 2018-03-28 00:36:34,231 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_50.expElogbeta.npy with mmap=None 2018-03-28 00:36:34,235 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:36:34,236 : INFO : setting ignored attribute state to None 2018-03-28 00:36:34,236 : INFO : setting ignored attribute id2word to None 2018-03-28 00:36:34,236 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_50 2018-03-28 00:36:34,237 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_50.state 2018-03-28 00:36:34,259 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_50.state 2018-03-28 00:36:34,286 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:36:34,335 : INFO : 1 batches submitted to accumulate stats from 64 documents (4982 virtual) 2018-03-28 00:36:34,342 : INFO : 2 batches submitted to accumulate stats from 128 documents (10157 virtual) 2018-03-28 00:36:34,350 : INFO : 3 batches submitted to accumulate stats from 192 documents (15057 virtual) 2018-03-28 00:36:34,357 : INFO : 4 batches submitted to accumulate stats from 256 documents (19941 virtual) 2018-03-28 00:36:34,365 : INFO : 5 batches submitted to accumulate stats from 320 documents (25048 virtual) 2018-03-28 00:36:34,372 : INFO : 6 batches submitted to accumulate stats from 384 documents (30581 virtual) 2018-03-28 00:36:34,565 : INFO : 7 batches submitted to accumulate stats from 448 documents (35845 virtual) 2018-03-28 00:36:34,600 : INFO : 8 batches submitted to accumulate stats from 512 documents (40777 virtual) 2018-03-28 00:36:34,625 : INFO : 9 batches submitted to accumulate stats from 576 documents (45638 virtual) 2018-03-28 00:36:34,844 : INFO : 10 batches submitted to accumulate stats from 640 documents (51118 virtual) 2018-03-28 00:36:34,879 : INFO : 11 batches submitted to accumulate stats from 704 documents (56333 virtual) 2018-03-28 00:36:34,901 : INFO : 12 batches submitted to accumulate stats from 768 documents (60983 virtual) 2018-03-28 00:36:35,076 : INFO : 13 batches submitted to accumulate stats from 832 documents (66643 virtual) 2018-03-28 00:36:35,106 : INFO : 14 batches submitted to accumulate stats from 896 documents (72016 virtual) 2018-03-28 00:36:35,178 : INFO : 15 batches submitted to accumulate stats from 960 documents (77174 virtual) 2018-03-28 00:36:35,301 : INFO : 16 batches submitted to accumulate stats from 1024 documents (83175 virtual) 2018-03-28 00:36:35,342 : INFO : 17 batches submitted to accumulate stats from 1088 documents (88314 virtual) 2018-03-28 00:36:35,392 : INFO : 18 batches submitted to accumulate stats from 1152 documents (93141 virtual) 2018-03-28 00:36:35,570 : INFO : 19 batches submitted to accumulate stats from 1216 documents (97904 virtual) 2018-03-28 00:36:35,582 : INFO : 20 batches submitted to accumulate stats from 1280 documents (103726 virtual) 2018-03-28 00:36:35,655 : INFO : 21 batches submitted to accumulate stats from 1344 documents (108954 virtual) 2018-03-28 00:36:35,814 : INFO : 22 batches submitted to accumulate stats from 1408 documents (114386 virtual) 2018-03-28 00:36:35,834 : INFO : 23 batches submitted to accumulate stats from 1472 documents (120015 virtual) 2018-03-28 00:36:35,877 : INFO : 24 batches submitted to accumulate stats from 1536 documents (125036 virtual) 2018-03-28 00:36:36,046 : INFO : 25 batches submitted to accumulate stats from 1600 documents (130590 virtual) 2018-03-28 00:36:36,077 : INFO : 26 batches submitted to accumulate stats from 1664 documents (135538 virtual) 2018-03-28 00:36:36,125 : INFO : 27 batches submitted to accumulate stats from 1728 documents (140904 virtual) 2018-03-28 00:36:36,295 : INFO : 28 batches submitted to accumulate stats from 1792 documents (146046 virtual) 2018-03-28 00:36:36,335 : INFO : 29 batches submitted to accumulate stats from 1856 documents (151365 virtual) 2018-03-28 00:36:36,365 : INFO : 30 batches submitted to accumulate stats from 1920 documents (157100 virtual) 2018-03-28 00:36:36,537 : INFO : 31 batches submitted to accumulate stats from 1984 documents (162022 virtual) 2018-03-28 00:36:36,557 : INFO : 32 batches submitted to accumulate stats from 2048 documents (167046 virtual) 2018-03-28 00:36:36,629 : INFO : 33 batches submitted to accumulate stats from 2112 documents (171918 virtual) 2018-03-28 00:36:36,777 : INFO : 34 batches submitted to accumulate stats from 2176 documents (177248 virtual) 2018-03-28 00:36:36,799 : INFO : 35 batches submitted to accumulate stats from 2240 documents (182757 virtual) 2018-03-28 00:36:36,898 : INFO : 36 batches submitted to accumulate stats from 2304 documents (187697 virtual) 2018-03-28 00:36:37,016 : INFO : 37 batches submitted to accumulate stats from 2368 documents (193401 virtual) 2018-03-28 00:36:37,056 : INFO : 38 batches submitted to accumulate stats from 2432 documents (199702 virtual) 2018-03-28 00:36:37,111 : INFO : 39 batches submitted to accumulate stats from 2496 documents (204584 virtual) 2018-03-28 00:36:37,252 : INFO : 40 batches submitted to accumulate stats from 2560 documents (209829 virtual) 2018-03-28 00:36:37,320 : INFO : 41 batches submitted to accumulate stats from 2624 documents (214652 virtual) 2018-03-28 00:36:37,351 : INFO : 42 batches submitted to accumulate stats from 2688 documents (219589 virtual) 2018-03-28 00:36:37,543 : INFO : 43 batches submitted to accumulate stats from 2752 documents (224425 virtual) 2018-03-28 00:36:37,569 : INFO : 44 batches submitted to accumulate stats from 2816 documents (228797 virtual) 2018-03-28 00:36:37,615 : INFO : 45 batches submitted to accumulate stats from 2880 documents (234337 virtual) 2018-03-28 00:36:37,792 : INFO : 46 batches submitted to accumulate stats from 2944 documents (239090 virtual) 2018-03-28 00:36:37,799 : INFO : 47 batches submitted to accumulate stats from 3008 documents (244163 virtual) 2018-03-28 00:36:37,862 : INFO : 48 batches submitted to accumulate stats from 3072 documents (249016 virtual) 2018-03-28 00:36:37,959 : INFO : 49 batches submitted to accumulate stats from 3136 documents (254021 virtual) 2018-03-28 00:36:38,020 : INFO : 50 batches submitted to accumulate stats from 3200 documents (259184 virtual) 2018-03-28 00:36:38,119 : INFO : 51 batches submitted to accumulate stats from 3264 documents (264615 virtual) 2018-03-28 00:36:38,201 : INFO : 52 batches submitted to accumulate stats from 3328 documents (269467 virtual) 2018-03-28 00:36:38,264 : INFO : 53 batches submitted to accumulate stats from 3392 documents (274797 virtual) 2018-03-28 00:36:38,364 : INFO : 54 batches submitted to accumulate stats from 3456 documents (279801 virtual) 2018-03-28 00:36:38,431 : INFO : 55 batches submitted to accumulate stats from 3520 documents (284585 virtual) 2018-03-28 00:36:38,496 : INFO : 56 batches submitted to accumulate stats from 3584 documents (289830 virtual) 2018-03-28 00:36:38,610 : INFO : 57 batches submitted to accumulate stats from 3648 documents (294996 virtual) 2018-03-28 00:36:38,654 : INFO : 58 batches submitted to accumulate stats from 3712 documents (300516 virtual) 2018-03-28 00:36:38,731 : INFO : 59 batches submitted to accumulate stats from 3776 documents (305642 virtual) 2018-03-28 00:36:38,854 : INFO : 60 batches submitted to accumulate stats from 3840 documents (310556 virtual) 2018-03-28 00:36:38,867 : INFO : 61 batches submitted to accumulate stats from 3904 documents (315832 virtual) 2018-03-28 00:36:38,994 : INFO : 62 batches submitted to accumulate stats from 3968 documents (320771 virtual) 2018-03-28 00:36:39,089 : INFO : 63 batches submitted to accumulate stats from 4032 documents (325491 virtual) 2018-03-28 00:36:39,108 : INFO : 64 batches submitted to accumulate stats from 4096 documents (330901 virtual) 2018-03-28 00:36:39,255 : INFO : 65 batches submitted to accumulate stats from 4160 documents (335837 virtual) 2018-03-28 00:36:39,320 : INFO : 66 batches submitted to accumulate stats from 4224 documents (341359 virtual) 2018-03-28 00:36:39,338 : INFO : 67 batches submitted to accumulate stats from 4288 documents (346349 virtual) 2018-03-28 00:36:39,496 : INFO : 68 batches submitted to accumulate stats from 4352 documents (351390 virtual) 2018-03-28 00:36:39,528 : INFO : 69 batches submitted to accumulate stats from 4416 documents (356421 virtual) 2018-03-28 00:36:39,598 : INFO : 70 batches submitted to accumulate stats from 4480 documents (361639 virtual) 2018-03-28 00:36:39,721 : INFO : 71 batches submitted to accumulate stats from 4544 documents (366641 virtual) 2018-03-28 00:36:39,802 : INFO : 72 batches submitted to accumulate stats from 4608 documents (371577 virtual) 2018-03-28 00:36:39,809 : INFO : 73 batches submitted to accumulate stats from 4672 documents (376647 virtual) 2018-03-28 00:36:39,964 : INFO : 74 batches submitted to accumulate stats from 4736 documents (382212 virtual) 2018-03-28 00:36:40,036 : INFO : 75 batches submitted to accumulate stats from 4800 documents (387502 virtual) 2018-03-28 00:36:40,054 : INFO : 76 batches submitted to accumulate stats from 4864 documents (392352 virtual) 2018-03-28 00:36:40,191 : INFO : 77 batches submitted to accumulate stats from 4928 documents (397254 virtual) 2018-03-28 00:36:40,273 : INFO : 78 batches submitted to accumulate stats from 4992 documents (402785 virtual) 2018-03-28 00:36:40,294 : INFO : 79 batches submitted to accumulate stats from 5056 documents (407587 virtual) 2018-03-28 00:36:40,411 : INFO : 80 batches submitted to accumulate stats from 5120 documents (412916 virtual) 2018-03-28 00:36:40,516 : INFO : 81 batches submitted to accumulate stats from 5184 documents (418346 virtual) 2018-03-28 00:36:40,549 : INFO : 82 batches submitted to accumulate stats from 5248 documents (423702 virtual) 2018-03-28 00:36:40,637 : INFO : 83 batches submitted to accumulate stats from 5312 documents (428973 virtual) 2018-03-28 00:36:40,757 : INFO : 84 batches submitted to accumulate stats from 5376 documents (434624 virtual) 2018-03-28 00:36:40,791 : INFO : 85 batches submitted to accumulate stats from 5440 documents (440230 virtual) 2018-03-28 00:36:40,901 : INFO : 86 batches submitted to accumulate stats from 5504 documents (446039 virtual) 2018-03-28 00:36:40,985 : INFO : 87 batches submitted to accumulate stats from 5568 documents (450373 virtual) 2018-03-28 00:36:41,029 : INFO : 88 batches submitted to accumulate stats from 5632 documents (455761 virtual) 2018-03-28 00:36:41,139 : INFO : 89 batches submitted to accumulate stats from 5696 documents (460897 virtual) 2018-03-28 00:36:41,271 : INFO : 90 batches submitted to accumulate stats from 5760 documents (466429 virtual) 2018-03-28 00:36:41,287 : INFO : 91 batches submitted to accumulate stats from 5824 documents (471884 virtual) 2018-03-28 00:36:41,384 : INFO : 92 batches submitted to accumulate stats from 5888 documents (477235 virtual) 2018-03-28 00:36:41,485 : INFO : 93 batches submitted to accumulate stats from 5952 documents (482371 virtual) 2018-03-28 00:36:41,532 : INFO : 94 batches submitted to accumulate stats from 6016 documents (487878 virtual) 2018-03-28 00:36:41,641 : INFO : 95 batches submitted to accumulate stats from 6080 documents (493691 virtual) 2018-03-28 00:36:41,746 : INFO : 96 batches submitted to accumulate stats from 6144 documents (498858 virtual) 2018-03-28 00:36:41,777 : INFO : 97 batches submitted to accumulate stats from 6208 documents (504486 virtual) 2018-03-28 00:36:41,886 : INFO : 98 batches submitted to accumulate stats from 6272 documents (509116 virtual) 2018-03-28 00:36:41,974 : INFO : 99 batches submitted to accumulate stats from 6336 documents (514222 virtual) 2018-03-28 00:36:42,005 : INFO : 100 batches submitted to accumulate stats from 6400 documents (519055 virtual) 2018-03-28 00:36:42,178 : INFO : 101 batches submitted to accumulate stats from 6464 documents (524415 virtual) 2018-03-28 00:36:42,204 : INFO : 102 batches submitted to accumulate stats from 6528 documents (529368 virtual) 2018-03-28 00:36:42,287 : INFO : 103 batches submitted to accumulate stats from 6592 documents (534395 virtual) 2018-03-28 00:36:42,400 : INFO : 104 batches submitted to accumulate stats from 6656 documents (539397 virtual) 2018-03-28 00:36:42,420 : INFO : 105 batches submitted to accumulate stats from 6720 documents (544799 virtual) 2018-03-28 00:36:42,519 : INFO : 106 batches submitted to accumulate stats from 6784 documents (549519 virtual) 2018-03-28 00:36:42,654 : INFO : 107 batches submitted to accumulate stats from 6848 documents (555051 virtual) 2018-03-28 00:36:42,661 : INFO : 108 batches submitted to accumulate stats from 6912 documents (559855 virtual) 2018-03-28 00:36:42,757 : INFO : 109 batches submitted to accumulate stats from 6976 documents (565802 virtual) 2018-03-28 00:36:42,865 : INFO : 110 batches submitted to accumulate stats from 7040 documents (570381 virtual) 2018-03-28 00:36:42,920 : INFO : 111 batches submitted to accumulate stats from 7104 documents (575391 virtual) 2018-03-28 00:36:43,000 : INFO : 112 batches submitted to accumulate stats from 7168 documents (580082 virtual) 2018-03-28 00:36:43,088 : INFO : 113 batches submitted to accumulate stats from 7232 documents (585352 virtual) 2018-03-28 00:36:43,179 : INFO : 114 batches submitted to accumulate stats from 7296 documents (590150 virtual) 2018-03-28 00:36:43,293 : INFO : 115 batches submitted to accumulate stats from 7360 documents (595359 virtual) 2018-03-28 00:36:43,298 : INFO : 116 batches submitted to accumulate stats from 7424 documents (600864 virtual) 2018-03-28 00:36:43,396 : INFO : 117 batches submitted to accumulate stats from 7488 documents (606299 virtual) 2018-03-28 00:36:43,494 : INFO : 118 batches submitted to accumulate stats from 7552 documents (611220 virtual) 2018-03-28 00:36:43,530 : INFO : 119 batches submitted to accumulate stats from 7616 documents (616369 virtual) 2018-03-28 00:36:43,642 : INFO : 120 batches submitted to accumulate stats from 7680 documents (621372 virtual) 2018-03-28 00:36:43,761 : INFO : 121 batches submitted to accumulate stats from 7744 documents (626419 virtual) 2018-03-28 00:36:43,776 : INFO : 122 batches submitted to accumulate stats from 7808 documents (632167 virtual) 2018-03-28 00:36:43,918 : INFO : 123 batches submitted to accumulate stats from 7872 documents (637255 virtual) 2018-03-28 00:36:43,968 : INFO : 124 batches submitted to accumulate stats from 7936 documents (642569 virtual) 2018-03-28 00:36:44,021 : INFO : 125 batches submitted to accumulate stats from 8000 documents (647585 virtual) 2018-03-28 00:36:44,154 : INFO : 126 batches submitted to accumulate stats from 8064 documents (647635 virtual) 2018-03-28 00:36:44,395 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:44,446 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:44,489 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:44,400 : INFO : accumulator serialized 2018-03-28 00:36:44,452 : INFO : accumulator serialized 2018-03-28 00:36:44,497 : INFO : accumulator serialized 2018-03-28 00:36:44,733 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:36:45,057 : INFO : accumulated word occurrence stats for 648333 virtual documents
In [50]:
tmfull_meta_score['c_uci'] = tmfull_meta_score.apply(lambda row: get_score(row.num_topic, dict_tmfull_meta, texts=text_tmfull_meta, corpus=corpus_tmfull_meta,
coherence='c_uci', model_dir='../src/tmfull_meta'),
axis=1)
2018-03-28 00:36:46,429 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_10 2018-03-28 00:36:46,432 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-28 00:36:46,434 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:36:46,435 : INFO : setting ignored attribute state to None 2018-03-28 00:36:46,436 : INFO : setting ignored attribute id2word to None 2018-03-28 00:36:46,436 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_10 2018-03-28 00:36:46,437 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_10.state 2018-03-28 00:36:46,443 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_10.state 2018-03-28 00:36:46,451 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:36:46,496 : INFO : 1 batches submitted to accumulate stats from 64 documents (4982 virtual) 2018-03-28 00:36:46,503 : INFO : 2 batches submitted to accumulate stats from 128 documents (10157 virtual) 2018-03-28 00:36:46,509 : INFO : 3 batches submitted to accumulate stats from 192 documents (15057 virtual) 2018-03-28 00:36:46,516 : INFO : 4 batches submitted to accumulate stats from 256 documents (20243 virtual) 2018-03-28 00:36:46,522 : INFO : 5 batches submitted to accumulate stats from 320 documents (25109 virtual) 2018-03-28 00:36:46,529 : INFO : 6 batches submitted to accumulate stats from 384 documents (30729 virtual) 2018-03-28 00:36:46,656 : INFO : 7 batches submitted to accumulate stats from 448 documents (36045 virtual) 2018-03-28 00:36:46,663 : INFO : 8 batches submitted to accumulate stats from 512 documents (40896 virtual) 2018-03-28 00:36:46,669 : INFO : 9 batches submitted to accumulate stats from 576 documents (45761 virtual) 2018-03-28 00:36:46,796 : INFO : 10 batches submitted to accumulate stats from 640 documents (51266 virtual) 2018-03-28 00:36:46,802 : INFO : 11 batches submitted to accumulate stats from 704 documents (56807 virtual) 2018-03-28 00:36:46,833 : INFO : 12 batches submitted to accumulate stats from 768 documents (61339 virtual) 2018-03-28 00:36:46,938 : INFO : 13 batches submitted to accumulate stats from 832 documents (67243 virtual) 2018-03-28 00:36:46,949 : INFO : 14 batches submitted to accumulate stats from 896 documents (72935 virtual) 2018-03-28 00:36:46,967 : INFO : 15 batches submitted to accumulate stats from 960 documents (78230 virtual) 2018-03-28 00:36:47,084 : INFO : 16 batches submitted to accumulate stats from 1024 documents (83975 virtual) 2018-03-28 00:36:47,090 : INFO : 17 batches submitted to accumulate stats from 1088 documents (89092 virtual) 2018-03-28 00:36:47,110 : INFO : 18 batches submitted to accumulate stats from 1152 documents (94203 virtual) 2018-03-28 00:36:47,228 : INFO : 19 batches submitted to accumulate stats from 1216 documents (99097 virtual) 2018-03-28 00:36:47,254 : INFO : 20 batches submitted to accumulate stats from 1280 documents (104698 virtual) 2018-03-28 00:36:47,264 : INFO : 21 batches submitted to accumulate stats from 1344 documents (110426 virtual) 2018-03-28 00:36:47,366 : INFO : 22 batches submitted to accumulate stats from 1408 documents (115651 virtual) 2018-03-28 00:36:47,398 : INFO : 23 batches submitted to accumulate stats from 1472 documents (121606 virtual) 2018-03-28 00:36:47,422 : INFO : 24 batches submitted to accumulate stats from 1536 documents (126947 virtual) 2018-03-28 00:36:47,484 : INFO : 25 batches submitted to accumulate stats from 1600 documents (132136 virtual) 2018-03-28 00:36:47,568 : INFO : 26 batches submitted to accumulate stats from 1664 documents (136767 virtual) 2018-03-28 00:36:47,576 : INFO : 27 batches submitted to accumulate stats from 1728 documents (142089 virtual) 2018-03-28 00:36:47,635 : INFO : 28 batches submitted to accumulate stats from 1792 documents (147605 virtual) 2018-03-28 00:36:47,710 : INFO : 29 batches submitted to accumulate stats from 1856 documents (153377 virtual) 2018-03-28 00:36:47,737 : INFO : 30 batches submitted to accumulate stats from 1920 documents (158669 virtual) 2018-03-28 00:36:47,770 : INFO : 31 batches submitted to accumulate stats from 1984 documents (163835 virtual) 2018-03-28 00:36:47,836 : INFO : 32 batches submitted to accumulate stats from 2048 documents (168727 virtual) 2018-03-28 00:36:47,896 : INFO : 33 batches submitted to accumulate stats from 2112 documents (173687 virtual) 2018-03-28 00:36:47,920 : INFO : 34 batches submitted to accumulate stats from 2176 documents (179414 virtual) 2018-03-28 00:36:47,991 : INFO : 35 batches submitted to accumulate stats from 2240 documents (184912 virtual) 2018-03-28 00:36:48,047 : INFO : 36 batches submitted to accumulate stats from 2304 documents (190083 virtual) 2018-03-28 00:36:48,062 : INFO : 37 batches submitted to accumulate stats from 2368 documents (195896 virtual) 2018-03-28 00:36:48,124 : INFO : 38 batches submitted to accumulate stats from 2432 documents (201785 virtual) 2018-03-28 00:36:48,189 : INFO : 39 batches submitted to accumulate stats from 2496 documents (206828 virtual) 2018-03-28 00:36:48,226 : INFO : 40 batches submitted to accumulate stats from 2560 documents (212216 virtual) 2018-03-28 00:36:48,280 : INFO : 41 batches submitted to accumulate stats from 2624 documents (216767 virtual) 2018-03-28 00:36:48,318 : INFO : 42 batches submitted to accumulate stats from 2688 documents (222176 virtual) 2018-03-28 00:36:48,365 : INFO : 43 batches submitted to accumulate stats from 2752 documents (226566 virtual) 2018-03-28 00:36:48,453 : INFO : 44 batches submitted to accumulate stats from 2816 documents (231686 virtual) 2018-03-28 00:36:48,465 : INFO : 45 batches submitted to accumulate stats from 2880 documents (236732 virtual) 2018-03-28 00:36:48,529 : INFO : 46 batches submitted to accumulate stats from 2944 documents (241514 virtual) 2018-03-28 00:36:48,604 : INFO : 47 batches submitted to accumulate stats from 3008 documents (246552 virtual) 2018-03-28 00:36:48,647 : INFO : 48 batches submitted to accumulate stats from 3072 documents (251553 virtual) 2018-03-28 00:36:48,682 : INFO : 49 batches submitted to accumulate stats from 3136 documents (256422 virtual) 2018-03-28 00:36:48,755 : INFO : 50 batches submitted to accumulate stats from 3200 documents (262545 virtual) 2018-03-28 00:36:48,774 : INFO : 51 batches submitted to accumulate stats from 3264 documents (267244 virtual) 2018-03-28 00:36:48,829 : INFO : 52 batches submitted to accumulate stats from 3328 documents (272652 virtual) 2018-03-28 00:36:48,901 : INFO : 53 batches submitted to accumulate stats from 3392 documents (277830 virtual) 2018-03-28 00:36:48,906 : INFO : 54 batches submitted to accumulate stats from 3456 documents (282897 virtual) 2018-03-28 00:36:48,956 : INFO : 55 batches submitted to accumulate stats from 3520 documents (288096 virtual) 2018-03-28 00:36:49,028 : INFO : 56 batches submitted to accumulate stats from 3584 documents (292972 virtual) 2018-03-28 00:36:49,064 : INFO : 57 batches submitted to accumulate stats from 3648 documents (298753 virtual) 2018-03-28 00:36:49,121 : INFO : 58 batches submitted to accumulate stats from 3712 documents (303929 virtual) 2018-03-28 00:36:49,159 : INFO : 59 batches submitted to accumulate stats from 3776 documents (308877 virtual) 2018-03-28 00:36:49,221 : INFO : 60 batches submitted to accumulate stats from 3840 documents (314413 virtual) 2018-03-28 00:36:49,251 : INFO : 61 batches submitted to accumulate stats from 3904 documents (319313 virtual) 2018-03-28 00:36:49,310 : INFO : 62 batches submitted to accumulate stats from 3968 documents (323950 virtual) 2018-03-28 00:36:49,378 : INFO : 63 batches submitted to accumulate stats from 4032 documents (329116 virtual) 2018-03-28 00:36:49,387 : INFO : 64 batches submitted to accumulate stats from 4096 documents (334615 virtual) 2018-03-28 00:36:49,436 : INFO : 65 batches submitted to accumulate stats from 4160 documents (340062 virtual) 2018-03-28 00:36:49,515 : INFO : 66 batches submitted to accumulate stats from 4224 documents (345064 virtual) 2018-03-28 00:36:49,559 : INFO : 67 batches submitted to accumulate stats from 4288 documents (350313 virtual) 2018-03-28 00:36:49,577 : INFO : 68 batches submitted to accumulate stats from 4352 documents (355588 virtual) 2018-03-28 00:36:49,634 : INFO : 69 batches submitted to accumulate stats from 4416 documents (360644 virtual) 2018-03-28 00:36:49,735 : INFO : 70 batches submitted to accumulate stats from 4480 documents (365848 virtual) 2018-03-28 00:36:49,742 : INFO : 71 batches submitted to accumulate stats from 4544 documents (370526 virtual) 2018-03-28 00:36:49,756 : INFO : 72 batches submitted to accumulate stats from 4608 documents (375897 virtual) 2018-03-28 00:36:49,879 : INFO : 73 batches submitted to accumulate stats from 4672 documents (381628 virtual) 2018-03-28 00:36:49,900 : INFO : 74 batches submitted to accumulate stats from 4736 documents (386903 virtual) 2018-03-28 00:36:49,906 : INFO : 75 batches submitted to accumulate stats from 4800 documents (391772 virtual) 2018-03-28 00:36:49,995 : INFO : 76 batches submitted to accumulate stats from 4864 documents (396642 virtual) 2018-03-28 00:36:50,043 : INFO : 77 batches submitted to accumulate stats from 4928 documents (402139 virtual) 2018-03-28 00:36:50,057 : INFO : 78 batches submitted to accumulate stats from 4992 documents (406987 virtual) 2018-03-28 00:36:50,134 : INFO : 79 batches submitted to accumulate stats from 5056 documents (412433 virtual) 2018-03-28 00:36:50,197 : INFO : 80 batches submitted to accumulate stats from 5120 documents (417859 virtual) 2018-03-28 00:36:50,209 : INFO : 81 batches submitted to accumulate stats from 5184 documents (423375 virtual) 2018-03-28 00:36:50,252 : INFO : 82 batches submitted to accumulate stats from 5248 documents (428518 virtual) 2018-03-28 00:36:50,349 : INFO : 83 batches submitted to accumulate stats from 5312 documents (434380 virtual) 2018-03-28 00:36:50,370 : INFO : 84 batches submitted to accumulate stats from 5376 documents (440012 virtual) 2018-03-28 00:36:50,376 : INFO : 85 batches submitted to accumulate stats from 5440 documents (445882 virtual) 2018-03-28 00:36:50,491 : INFO : 86 batches submitted to accumulate stats from 5504 documents (450244 virtual) 2018-03-28 00:36:50,517 : INFO : 87 batches submitted to accumulate stats from 5568 documents (455808 virtual) 2018-03-28 00:36:50,549 : INFO : 88 batches submitted to accumulate stats from 5632 documents (460886 virtual) 2018-03-28 00:36:50,623 : INFO : 89 batches submitted to accumulate stats from 5696 documents (466358 virtual) 2018-03-28 00:36:50,689 : INFO : 90 batches submitted to accumulate stats from 5760 documents (471842 virtual) 2018-03-28 00:36:50,722 : INFO : 91 batches submitted to accumulate stats from 5824 documents (477442 virtual) 2018-03-28 00:36:50,743 : INFO : 92 batches submitted to accumulate stats from 5888 documents (482773 virtual) 2018-03-28 00:36:50,839 : INFO : 93 batches submitted to accumulate stats from 5952 documents (488229 virtual) 2018-03-28 00:36:50,876 : INFO : 94 batches submitted to accumulate stats from 6016 documents (493653 virtual) 2018-03-28 00:36:50,888 : INFO : 95 batches submitted to accumulate stats from 6080 documents (499329 virtual) 2018-03-28 00:36:51,011 : INFO : 96 batches submitted to accumulate stats from 6144 documents (504582 virtual) 2018-03-28 00:36:51,018 : INFO : 97 batches submitted to accumulate stats from 6208 documents (509569 virtual) 2018-03-28 00:36:51,030 : INFO : 98 batches submitted to accumulate stats from 6272 documents (514533 virtual) 2018-03-28 00:36:51,146 : INFO : 99 batches submitted to accumulate stats from 6336 documents (519517 virtual) 2018-03-28 00:36:51,177 : INFO : 100 batches submitted to accumulate stats from 6400 documents (524779 virtual) 2018-03-28 00:36:51,182 : INFO : 101 batches submitted to accumulate stats from 6464 documents (529613 virtual) 2018-03-28 00:36:51,291 : INFO : 102 batches submitted to accumulate stats from 6528 documents (534611 virtual) 2018-03-28 00:36:51,313 : INFO : 103 batches submitted to accumulate stats from 6592 documents (540049 virtual) 2018-03-28 00:36:51,325 : INFO : 104 batches submitted to accumulate stats from 6656 documents (545389 virtual) 2018-03-28 00:36:51,436 : INFO : 105 batches submitted to accumulate stats from 6720 documents (550182 virtual) 2018-03-28 00:36:51,442 : INFO : 106 batches submitted to accumulate stats from 6784 documents (555700 virtual) 2018-03-28 00:36:51,477 : INFO : 107 batches submitted to accumulate stats from 6848 documents (560813 virtual) 2018-03-28 00:36:51,560 : INFO : 108 batches submitted to accumulate stats from 6912 documents (566484 virtual) 2018-03-28 00:36:51,601 : INFO : 109 batches submitted to accumulate stats from 6976 documents (571428 virtual) 2018-03-28 00:36:51,625 : INFO : 110 batches submitted to accumulate stats from 7040 documents (576165 virtual) 2018-03-28 00:36:51,688 : INFO : 111 batches submitted to accumulate stats from 7104 documents (580954 virtual) 2018-03-28 00:36:51,760 : INFO : 112 batches submitted to accumulate stats from 7168 documents (586430 virtual) 2018-03-28 00:36:51,769 : INFO : 113 batches submitted to accumulate stats from 7232 documents (591062 virtual) 2018-03-28 00:36:51,849 : INFO : 114 batches submitted to accumulate stats from 7296 documents (596601 virtual) 2018-03-28 00:36:51,881 : INFO : 115 batches submitted to accumulate stats from 7360 documents (602146 virtual) 2018-03-28 00:36:51,920 : INFO : 116 batches submitted to accumulate stats from 7424 documents (607081 virtual) 2018-03-28 00:36:51,978 : INFO : 117 batches submitted to accumulate stats from 7488 documents (612148 virtual) 2018-03-28 00:36:52,030 : INFO : 118 batches submitted to accumulate stats from 7552 documents (617413 virtual) 2018-03-28 00:36:52,071 : INFO : 119 batches submitted to accumulate stats from 7616 documents (622732 virtual) 2018-03-28 00:36:52,128 : INFO : 120 batches submitted to accumulate stats from 7680 documents (627979 virtual) 2018-03-28 00:36:52,187 : INFO : 121 batches submitted to accumulate stats from 7744 documents (633848 virtual) 2018-03-28 00:36:52,214 : INFO : 122 batches submitted to accumulate stats from 7808 documents (638824 virtual) 2018-03-28 00:36:52,292 : INFO : 123 batches submitted to accumulate stats from 7872 documents (643881 virtual) 2018-03-28 00:36:52,348 : INFO : 124 batches submitted to accumulate stats from 7936 documents (647126 virtual) 2018-03-28 00:36:52,532 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:52,540 : INFO : accumulator serialized 2018-03-28 00:36:52,588 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:52,594 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:52,592 : INFO : accumulator serialized 2018-03-28 00:36:52,601 : INFO : accumulator serialized 2018-03-28 00:36:52,644 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:36:52,680 : INFO : accumulated word occurrence stats for 647659 virtual documents 2018-03-28 00:36:52,821 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_15 2018-03-28 00:36:52,823 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_15.expElogbeta.npy with mmap=None 2018-03-28 00:36:52,826 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:36:52,827 : INFO : setting ignored attribute state to None 2018-03-28 00:36:52,827 : INFO : setting ignored attribute id2word to None 2018-03-28 00:36:52,828 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_15 2018-03-28 00:36:52,828 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_15.state 2018-03-28 00:36:52,837 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_15.state 2018-03-28 00:36:52,848 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:36:52,892 : INFO : 1 batches submitted to accumulate stats from 64 documents (4982 virtual) 2018-03-28 00:36:52,900 : INFO : 2 batches submitted to accumulate stats from 128 documents (10157 virtual) 2018-03-28 00:36:52,906 : INFO : 3 batches submitted to accumulate stats from 192 documents (15057 virtual) 2018-03-28 00:36:52,914 : INFO : 4 batches submitted to accumulate stats from 256 documents (19941 virtual) 2018-03-28 00:36:52,921 : INFO : 5 batches submitted to accumulate stats from 320 documents (25048 virtual) 2018-03-28 00:36:52,927 : INFO : 6 batches submitted to accumulate stats from 384 documents (30581 virtual) 2018-03-28 00:36:53,067 : INFO : 7 batches submitted to accumulate stats from 448 documents (35845 virtual) 2018-03-28 00:36:53,074 : INFO : 8 batches submitted to accumulate stats from 512 documents (40777 virtual) 2018-03-28 00:36:53,082 : INFO : 9 batches submitted to accumulate stats from 576 documents (45638 virtual) 2018-03-28 00:36:53,225 : INFO : 10 batches submitted to accumulate stats from 640 documents (51118 virtual) 2018-03-28 00:36:53,244 : INFO : 11 batches submitted to accumulate stats from 704 documents (56409 virtual) 2018-03-28 00:36:53,250 : INFO : 12 batches submitted to accumulate stats from 768 documents (61058 virtual) 2018-03-28 00:36:53,389 : INFO : 13 batches submitted to accumulate stats from 832 documents (66886 virtual) 2018-03-28 00:36:53,395 : INFO : 14 batches submitted to accumulate stats from 896 documents (72586 virtual) 2018-03-28 00:36:53,410 : INFO : 15 batches submitted to accumulate stats from 960 documents (77877 virtual) 2018-03-28 00:36:53,556 : INFO : 16 batches submitted to accumulate stats from 1024 documents (83673 virtual) 2018-03-28 00:36:53,572 : INFO : 17 batches submitted to accumulate stats from 1088 documents (88804 virtual) 2018-03-28 00:36:53,587 : INFO : 18 batches submitted to accumulate stats from 1152 documents (93954 virtual) 2018-03-28 00:36:53,733 : INFO : 19 batches submitted to accumulate stats from 1216 documents (98617 virtual) 2018-03-28 00:36:53,743 : INFO : 20 batches submitted to accumulate stats from 1280 documents (104250 virtual) 2018-03-28 00:36:53,762 : INFO : 21 batches submitted to accumulate stats from 1344 documents (109784 virtual) 2018-03-28 00:36:53,900 : INFO : 22 batches submitted to accumulate stats from 1408 documents (114780 virtual) 2018-03-28 00:36:53,912 : INFO : 23 batches submitted to accumulate stats from 1472 documents (120626 virtual) 2018-03-28 00:36:53,921 : INFO : 24 batches submitted to accumulate stats from 1536 documents (126158 virtual) 2018-03-28 00:36:54,053 : INFO : 25 batches submitted to accumulate stats from 1600 documents (131438 virtual) 2018-03-28 00:36:54,079 : INFO : 26 batches submitted to accumulate stats from 1664 documents (135958 virtual) 2018-03-28 00:36:54,096 : INFO : 27 batches submitted to accumulate stats from 1728 documents (141622 virtual) 2018-03-28 00:36:54,212 : INFO : 28 batches submitted to accumulate stats from 1792 documents (146695 virtual) 2018-03-28 00:36:54,257 : INFO : 29 batches submitted to accumulate stats from 1856 documents (152488 virtual) 2018-03-28 00:36:54,271 : INFO : 30 batches submitted to accumulate stats from 1920 documents (158014 virtual) 2018-03-28 00:36:54,372 : INFO : 31 batches submitted to accumulate stats from 1984 documents (163034 virtual) 2018-03-28 00:36:54,391 : INFO : 32 batches submitted to accumulate stats from 2048 documents (168065 virtual) 2018-03-28 00:36:54,459 : INFO : 33 batches submitted to accumulate stats from 2112 documents (172956 virtual) 2018-03-28 00:36:54,528 : INFO : 34 batches submitted to accumulate stats from 2176 documents (178439 virtual) 2018-03-28 00:36:54,551 : INFO : 35 batches submitted to accumulate stats from 2240 documents (183952 virtual) 2018-03-28 00:36:54,650 : INFO : 36 batches submitted to accumulate stats from 2304 documents (189020 virtual) 2018-03-28 00:36:54,694 : INFO : 37 batches submitted to accumulate stats from 2368 documents (195016 virtual) 2018-03-28 00:36:54,718 : INFO : 38 batches submitted to accumulate stats from 2432 documents (200909 virtual) 2018-03-28 00:36:54,802 : INFO : 39 batches submitted to accumulate stats from 2496 documents (205941 virtual) 2018-03-28 00:36:54,877 : INFO : 40 batches submitted to accumulate stats from 2560 documents (211023 virtual) 2018-03-28 00:36:54,881 : INFO : 41 batches submitted to accumulate stats from 2624 documents (215691 virtual) 2018-03-28 00:36:54,939 : INFO : 42 batches submitted to accumulate stats from 2688 documents (221215 virtual) 2018-03-28 00:36:55,035 : INFO : 43 batches submitted to accumulate stats from 2752 documents (225418 virtual) 2018-03-28 00:36:55,078 : INFO : 44 batches submitted to accumulate stats from 2816 documents (230340 virtual) 2018-03-28 00:36:55,126 : INFO : 45 batches submitted to accumulate stats from 2880 documents (235829 virtual) 2018-03-28 00:36:55,191 : INFO : 46 batches submitted to accumulate stats from 2944 documents (240875 virtual) 2018-03-28 00:36:55,215 : INFO : 47 batches submitted to accumulate stats from 3008 documents (245402 virtual) 2018-03-28 00:36:55,284 : INFO : 48 batches submitted to accumulate stats from 3072 documents (250356 virtual) 2018-03-28 00:36:55,324 : INFO : 49 batches submitted to accumulate stats from 3136 documents (255295 virtual) 2018-03-28 00:36:55,368 : INFO : 50 batches submitted to accumulate stats from 3200 documents (261177 virtual) 2018-03-28 00:36:55,454 : INFO : 51 batches submitted to accumulate stats from 3264 documents (266149 virtual) 2018-03-28 00:36:55,486 : INFO : 52 batches submitted to accumulate stats from 3328 documents (271374 virtual) 2018-03-28 00:36:55,506 : INFO : 53 batches submitted to accumulate stats from 3392 documents (276331 virtual) 2018-03-28 00:36:55,614 : INFO : 54 batches submitted to accumulate stats from 3456 documents (281542 virtual) 2018-03-28 00:36:55,660 : INFO : 55 batches submitted to accumulate stats from 3520 documents (286462 virtual) 2018-03-28 00:36:55,668 : INFO : 56 batches submitted to accumulate stats from 3584 documents (291443 virtual) 2018-03-28 00:36:55,749 : INFO : 57 batches submitted to accumulate stats from 3648 documents (297042 virtual) 2018-03-28 00:36:55,819 : INFO : 58 batches submitted to accumulate stats from 3712 documents (302255 virtual) 2018-03-28 00:36:55,831 : INFO : 59 batches submitted to accumulate stats from 3776 documents (307481 virtual) 2018-03-28 00:36:55,923 : INFO : 60 batches submitted to accumulate stats from 3840 documents (312618 virtual) 2018-03-28 00:36:55,969 : INFO : 61 batches submitted to accumulate stats from 3904 documents (317936 virtual) 2018-03-28 00:36:55,998 : INFO : 62 batches submitted to accumulate stats from 3968 documents (322678 virtual) 2018-03-28 00:36:56,071 : INFO : 63 batches submitted to accumulate stats from 4032 documents (327278 virtual) 2018-03-28 00:36:56,142 : INFO : 64 batches submitted to accumulate stats from 4096 documents (332843 virtual) 2018-03-28 00:36:56,169 : INFO : 65 batches submitted to accumulate stats from 4160 documents (337816 virtual) 2018-03-28 00:36:56,238 : INFO : 66 batches submitted to accumulate stats from 4224 documents (343147 virtual) 2018-03-28 00:36:56,309 : INFO : 67 batches submitted to accumulate stats from 4288 documents (348325 virtual) 2018-03-28 00:36:56,315 : INFO : 68 batches submitted to accumulate stats from 4352 documents (353585 virtual) 2018-03-28 00:36:56,390 : INFO : 69 batches submitted to accumulate stats from 4416 documents (358769 virtual) 2018-03-28 00:36:56,484 : INFO : 70 batches submitted to accumulate stats from 4480 documents (363761 virtual) 2018-03-28 00:36:56,488 : INFO : 71 batches submitted to accumulate stats from 4544 documents (368526 virtual) 2018-03-28 00:36:56,531 : INFO : 72 batches submitted to accumulate stats from 4608 documents (373588 virtual) 2018-03-28 00:36:56,638 : INFO : 73 batches submitted to accumulate stats from 4672 documents (378928 virtual) 2018-03-28 00:36:56,668 : INFO : 74 batches submitted to accumulate stats from 4736 documents (384334 virtual) 2018-03-28 00:36:56,685 : INFO : 75 batches submitted to accumulate stats from 4800 documents (389837 virtual) 2018-03-28 00:36:56,783 : INFO : 76 batches submitted to accumulate stats from 4864 documents (394001 virtual) 2018-03-28 00:36:56,812 : INFO : 77 batches submitted to accumulate stats from 4928 documents (399863 virtual) 2018-03-28 00:36:56,848 : INFO : 78 batches submitted to accumulate stats from 4992 documents (404651 virtual) 2018-03-28 00:36:56,964 : INFO : 79 batches submitted to accumulate stats from 5056 documents (409835 virtual) 2018-03-28 00:36:56,976 : INFO : 80 batches submitted to accumulate stats from 5120 documents (415102 virtual) 2018-03-28 00:36:57,020 : INFO : 81 batches submitted to accumulate stats from 5184 documents (420818 virtual) 2018-03-28 00:36:57,103 : INFO : 82 batches submitted to accumulate stats from 5248 documents (426106 virtual) 2018-03-28 00:36:57,139 : INFO : 83 batches submitted to accumulate stats from 5312 documents (431416 virtual) 2018-03-28 00:36:57,194 : INFO : 84 batches submitted to accumulate stats from 5376 documents (437077 virtual) 2018-03-28 00:36:57,273 : INFO : 85 batches submitted to accumulate stats from 5440 documents (442566 virtual) 2018-03-28 00:36:57,286 : INFO : 86 batches submitted to accumulate stats from 5504 documents (448354 virtual) 2018-03-28 00:36:57,351 : INFO : 87 batches submitted to accumulate stats from 5568 documents (453066 virtual) 2018-03-28 00:36:57,450 : INFO : 88 batches submitted to accumulate stats from 5632 documents (458371 virtual) 2018-03-28 00:36:57,462 : INFO : 89 batches submitted to accumulate stats from 5696 documents (463517 virtual) 2018-03-28 00:36:57,515 : INFO : 90 batches submitted to accumulate stats from 5760 documents (469153 virtual) 2018-03-28 00:36:57,618 : INFO : 91 batches submitted to accumulate stats from 5824 documents (474495 virtual) 2018-03-28 00:36:57,649 : INFO : 92 batches submitted to accumulate stats from 5888 documents (479818 virtual) 2018-03-28 00:36:57,672 : INFO : 93 batches submitted to accumulate stats from 5952 documents (485199 virtual) 2018-03-28 00:36:57,763 : INFO : 94 batches submitted to accumulate stats from 6016 documents (491103 virtual) 2018-03-28 00:36:57,802 : INFO : 95 batches submitted to accumulate stats from 6080 documents (496453 virtual) 2018-03-28 00:36:57,851 : INFO : 96 batches submitted to accumulate stats from 6144 documents (501470 virtual) 2018-03-28 00:36:57,937 : INFO : 97 batches submitted to accumulate stats from 6208 documents (506849 virtual) 2018-03-28 00:36:57,985 : INFO : 98 batches submitted to accumulate stats from 6272 documents (511923 virtual) 2018-03-28 00:36:58,022 : INFO : 99 batches submitted to accumulate stats from 6336 documents (516767 virtual) 2018-03-28 00:36:58,117 : INFO : 100 batches submitted to accumulate stats from 6400 documents (521852 virtual) 2018-03-28 00:36:58,142 : INFO : 101 batches submitted to accumulate stats from 6464 documents (526866 virtual) 2018-03-28 00:36:58,187 : INFO : 102 batches submitted to accumulate stats from 6528 documents (532029 virtual) 2018-03-28 00:36:58,278 : INFO : 103 batches submitted to accumulate stats from 6592 documents (537379 virtual) 2018-03-28 00:36:58,302 : INFO : 104 batches submitted to accumulate stats from 6656 documents (542324 virtual) 2018-03-28 00:36:58,353 : INFO : 105 batches submitted to accumulate stats from 6720 documents (547604 virtual) 2018-03-28 00:36:58,438 : INFO : 106 batches submitted to accumulate stats from 6784 documents (552876 virtual) 2018-03-28 00:36:58,465 : INFO : 107 batches submitted to accumulate stats from 6848 documents (557940 virtual) 2018-03-28 00:36:58,499 : INFO : 108 batches submitted to accumulate stats from 6912 documents (563355 virtual) 2018-03-28 00:36:58,603 : INFO : 109 batches submitted to accumulate stats from 6976 documents (568839 virtual) 2018-03-28 00:36:58,615 : INFO : 110 batches submitted to accumulate stats from 7040 documents (573876 virtual) 2018-03-28 00:36:58,674 : INFO : 111 batches submitted to accumulate stats from 7104 documents (578111 virtual) 2018-03-28 00:36:58,754 : INFO : 112 batches submitted to accumulate stats from 7168 documents (583423 virtual) 2018-03-28 00:36:58,794 : INFO : 113 batches submitted to accumulate stats from 7232 documents (588643 virtual) 2018-03-28 00:36:58,832 : INFO : 114 batches submitted to accumulate stats from 7296 documents (593440 virtual) 2018-03-28 00:36:58,930 : INFO : 115 batches submitted to accumulate stats from 7360 documents (599104 virtual) 2018-03-28 00:36:58,939 : INFO : 116 batches submitted to accumulate stats from 7424 documents (604164 virtual) 2018-03-28 00:36:58,976 : INFO : 117 batches submitted to accumulate stats from 7488 documents (609254 virtual) 2018-03-28 00:36:59,092 : INFO : 118 batches submitted to accumulate stats from 7552 documents (614718 virtual) 2018-03-28 00:36:59,097 : INFO : 119 batches submitted to accumulate stats from 7616 documents (619684 virtual) 2018-03-28 00:36:59,174 : INFO : 120 batches submitted to accumulate stats from 7680 documents (624868 virtual) 2018-03-28 00:36:59,235 : INFO : 121 batches submitted to accumulate stats from 7744 documents (630169 virtual) 2018-03-28 00:36:59,274 : INFO : 122 batches submitted to accumulate stats from 7808 documents (635785 virtual) 2018-03-28 00:36:59,319 : INFO : 123 batches submitted to accumulate stats from 7872 documents (640705 virtual) 2018-03-28 00:36:59,414 : INFO : 124 batches submitted to accumulate stats from 7936 documents (646061 virtual) 2018-03-28 00:36:59,417 : INFO : 125 batches submitted to accumulate stats from 8000 documents (647394 virtual) 2018-03-28 00:36:59,644 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:59,657 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:59,650 : INFO : accumulator serialized 2018-03-28 00:36:59,707 : INFO : serializing accumulator to return to master... 2018-03-28 00:36:59,665 : INFO : accumulator serialized 2018-03-28 00:36:59,713 : INFO : accumulator serialized 2018-03-28 00:36:59,777 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:36:59,843 : INFO : accumulated word occurrence stats for 648012 virtual documents 2018-03-28 00:37:00,054 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_20 2018-03-28 00:37:00,056 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_20.expElogbeta.npy with mmap=None 2018-03-28 00:37:00,058 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:37:00,059 : INFO : setting ignored attribute state to None 2018-03-28 00:37:00,059 : INFO : setting ignored attribute id2word to None 2018-03-28 00:37:00,060 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_20 2018-03-28 00:37:00,061 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_20.state 2018-03-28 00:37:00,071 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_20.state 2018-03-28 00:37:00,084 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:37:00,129 : INFO : 1 batches submitted to accumulate stats from 64 documents (4982 virtual) 2018-03-28 00:37:00,137 : INFO : 2 batches submitted to accumulate stats from 128 documents (10157 virtual) 2018-03-28 00:37:00,143 : INFO : 3 batches submitted to accumulate stats from 192 documents (15057 virtual) 2018-03-28 00:37:00,150 : INFO : 4 batches submitted to accumulate stats from 256 documents (19941 virtual) 2018-03-28 00:37:00,157 : INFO : 5 batches submitted to accumulate stats from 320 documents (25048 virtual) 2018-03-28 00:37:00,164 : INFO : 6 batches submitted to accumulate stats from 384 documents (30581 virtual) 2018-03-28 00:37:00,305 : INFO : 7 batches submitted to accumulate stats from 448 documents (35845 virtual) 2018-03-28 00:37:00,318 : INFO : 8 batches submitted to accumulate stats from 512 documents (40777 virtual) 2018-03-28 00:37:00,354 : INFO : 9 batches submitted to accumulate stats from 576 documents (45638 virtual) 2018-03-28 00:37:00,469 : INFO : 10 batches submitted to accumulate stats from 640 documents (51118 virtual) 2018-03-28 00:37:00,501 : INFO : 11 batches submitted to accumulate stats from 704 documents (56407 virtual) 2018-03-28 00:37:00,513 : INFO : 12 batches submitted to accumulate stats from 768 documents (61064 virtual) 2018-03-28 00:37:00,644 : INFO : 13 batches submitted to accumulate stats from 832 documents (66752 virtual) 2018-03-28 00:37:00,671 : INFO : 14 batches submitted to accumulate stats from 896 documents (72410 virtual) 2018-03-28 00:37:00,712 : INFO : 15 batches submitted to accumulate stats from 960 documents (77897 virtual) 2018-03-28 00:37:00,803 : INFO : 16 batches submitted to accumulate stats from 1024 documents (83693 virtual) 2018-03-28 00:37:00,828 : INFO : 17 batches submitted to accumulate stats from 1088 documents (88824 virtual) 2018-03-28 00:37:00,907 : INFO : 18 batches submitted to accumulate stats from 1152 documents (93843 virtual) 2018-03-28 00:37:00,996 : INFO : 19 batches submitted to accumulate stats from 1216 documents (98507 virtual) 2018-03-28 00:37:01,012 : INFO : 20 batches submitted to accumulate stats from 1280 documents (104200 virtual) 2018-03-28 00:37:01,088 : INFO : 21 batches submitted to accumulate stats from 1344 documents (109596 virtual) 2018-03-28 00:37:01,186 : INFO : 22 batches submitted to accumulate stats from 1408 documents (114736 virtual) 2018-03-28 00:37:01,197 : INFO : 23 batches submitted to accumulate stats from 1472 documents (120412 virtual) 2018-03-28 00:37:01,256 : INFO : 24 batches submitted to accumulate stats from 1536 documents (125964 virtual) 2018-03-28 00:37:01,332 : INFO : 25 batches submitted to accumulate stats from 1600 documents (131215 virtual) 2018-03-28 00:37:01,386 : INFO : 26 batches submitted to accumulate stats from 1664 documents (135773 virtual) 2018-03-28 00:37:01,424 : INFO : 27 batches submitted to accumulate stats from 1728 documents (141410 virtual) 2018-03-28 00:37:01,526 : INFO : 28 batches submitted to accumulate stats from 1792 documents (146517 virtual) 2018-03-28 00:37:01,552 : INFO : 29 batches submitted to accumulate stats from 1856 documents (152359 virtual) 2018-03-28 00:37:01,631 : INFO : 30 batches submitted to accumulate stats from 1920 documents (157670 virtual) 2018-03-28 00:37:01,691 : INFO : 31 batches submitted to accumulate stats from 1984 documents (162805 virtual) 2018-03-28 00:37:01,723 : INFO : 32 batches submitted to accumulate stats from 2048 documents (167775 virtual) 2018-03-28 00:37:01,811 : INFO : 33 batches submitted to accumulate stats from 2112 documents (172403 virtual) 2018-03-28 00:37:01,851 : INFO : 34 batches submitted to accumulate stats from 2176 documents (177998 virtual) 2018-03-28 00:37:01,928 : INFO : 35 batches submitted to accumulate stats from 2240 documents (183591 virtual) 2018-03-28 00:37:01,995 : INFO : 36 batches submitted to accumulate stats from 2304 documents (188745 virtual) 2018-03-28 00:37:02,011 : INFO : 37 batches submitted to accumulate stats from 2368 documents (194385 virtual) 2018-03-28 00:37:02,088 : INFO : 38 batches submitted to accumulate stats from 2432 documents (200561 virtual) 2018-03-28 00:37:02,162 : INFO : 39 batches submitted to accumulate stats from 2496 documents (205501 virtual) 2018-03-28 00:37:02,205 : INFO : 40 batches submitted to accumulate stats from 2560 documents (210729 virtual) 2018-03-28 00:37:02,298 : INFO : 41 batches submitted to accumulate stats from 2624 documents (215226 virtual) 2018-03-28 00:37:02,351 : INFO : 42 batches submitted to accumulate stats from 2688 documents (220560 virtual) 2018-03-28 00:37:02,414 : INFO : 43 batches submitted to accumulate stats from 2752 documents (225093 virtual) 2018-03-28 00:37:02,514 : INFO : 44 batches submitted to accumulate stats from 2816 documents (229832 virtual) 2018-03-28 00:37:02,521 : INFO : 45 batches submitted to accumulate stats from 2880 documents (235373 virtual) 2018-03-28 00:37:02,624 : INFO : 46 batches submitted to accumulate stats from 2944 documents (240140 virtual) 2018-03-28 00:37:02,659 : INFO : 47 batches submitted to accumulate stats from 3008 documents (244952 virtual) 2018-03-28 00:37:02,716 : INFO : 48 batches submitted to accumulate stats from 3072 documents (249877 virtual) 2018-03-28 00:37:02,760 : INFO : 49 batches submitted to accumulate stats from 3136 documents (254875 virtual) 2018-03-28 00:37:02,814 : INFO : 50 batches submitted to accumulate stats from 3200 documents (260445 virtual) 2018-03-28 00:37:02,900 : INFO : 51 batches submitted to accumulate stats from 3264 documents (265513 virtual) 2018-03-28 00:37:02,907 : INFO : 52 batches submitted to accumulate stats from 3328 documents (270743 virtual) 2018-03-28 00:37:02,999 : INFO : 53 batches submitted to accumulate stats from 3392 documents (275753 virtual) 2018-03-28 00:37:03,058 : INFO : 54 batches submitted to accumulate stats from 3456 documents (280784 virtual) 2018-03-28 00:37:03,074 : INFO : 55 batches submitted to accumulate stats from 3520 documents (286064 virtual) 2018-03-28 00:37:03,173 : INFO : 56 batches submitted to accumulate stats from 3584 documents (290769 virtual) 2018-03-28 00:37:03,232 : INFO : 57 batches submitted to accumulate stats from 3648 documents (296488 virtual) 2018-03-28 00:37:03,268 : INFO : 58 batches submitted to accumulate stats from 3712 documents (301421 virtual) 2018-03-28 00:37:03,319 : INFO : 59 batches submitted to accumulate stats from 3776 documents (306734 virtual) 2018-03-28 00:37:03,414 : INFO : 60 batches submitted to accumulate stats from 3840 documents (311918 virtual) 2018-03-28 00:37:03,452 : INFO : 61 batches submitted to accumulate stats from 3904 documents (317212 virtual) 2018-03-28 00:37:03,476 : INFO : 62 batches submitted to accumulate stats from 3968 documents (322089 virtual) 2018-03-28 00:37:03,600 : INFO : 63 batches submitted to accumulate stats from 4032 documents (326781 virtual) 2018-03-28 00:37:03,606 : INFO : 64 batches submitted to accumulate stats from 4096 documents (332204 virtual) 2018-03-28 00:37:03,653 : INFO : 65 batches submitted to accumulate stats from 4160 documents (337312 virtual) 2018-03-28 00:37:03,768 : INFO : 66 batches submitted to accumulate stats from 4224 documents (342640 virtual) 2018-03-28 00:37:03,785 : INFO : 67 batches submitted to accumulate stats from 4288 documents (347711 virtual) 2018-03-28 00:37:03,832 : INFO : 68 batches submitted to accumulate stats from 4352 documents (352897 virtual) 2018-03-28 00:37:03,939 : INFO : 69 batches submitted to accumulate stats from 4416 documents (358011 virtual) 2018-03-28 00:37:03,986 : INFO : 70 batches submitted to accumulate stats from 4480 documents (363147 virtual) 2018-03-28 00:37:04,014 : INFO : 71 batches submitted to accumulate stats from 4544 documents (368013 virtual) 2018-03-28 00:37:04,098 : INFO : 72 batches submitted to accumulate stats from 4608 documents (373143 virtual) 2018-03-28 00:37:04,148 : INFO : 73 batches submitted to accumulate stats from 4672 documents (378189 virtual) 2018-03-28 00:37:04,188 : INFO : 74 batches submitted to accumulate stats from 4736 documents (383853 virtual) 2018-03-28 00:37:04,264 : INFO : 75 batches submitted to accumulate stats from 4800 documents (389123 virtual) 2018-03-28 00:37:04,322 : INFO : 76 batches submitted to accumulate stats from 4864 documents (393457 virtual) 2018-03-28 00:37:04,347 : INFO : 77 batches submitted to accumulate stats from 4928 documents (398948 virtual) 2018-03-28 00:37:04,447 : INFO : 78 batches submitted to accumulate stats from 4992 documents (403866 virtual) 2018-03-28 00:37:04,488 : INFO : 79 batches submitted to accumulate stats from 5056 documents (408996 virtual) 2018-03-28 00:37:04,526 : INFO : 80 batches submitted to accumulate stats from 5120 documents (414464 virtual) 2018-03-28 00:37:04,640 : INFO : 81 batches submitted to accumulate stats from 5184 documents (420250 virtual) 2018-03-28 00:37:04,653 : INFO : 82 batches submitted to accumulate stats from 5248 documents (425365 virtual) 2018-03-28 00:37:04,686 : INFO : 83 batches submitted to accumulate stats from 5312 documents (430578 virtual) 2018-03-28 00:37:04,803 : INFO : 84 batches submitted to accumulate stats from 5376 documents (436352 virtual) 2018-03-28 00:37:04,834 : INFO : 85 batches submitted to accumulate stats from 5440 documents (441886 virtual) 2018-03-28 00:37:04,883 : INFO : 86 batches submitted to accumulate stats from 5504 documents (447523 virtual) 2018-03-28 00:37:04,997 : INFO : 87 batches submitted to accumulate stats from 5568 documents (452242 virtual) 2018-03-28 00:37:05,012 : INFO : 88 batches submitted to accumulate stats from 5632 documents (457553 virtual) 2018-03-28 00:37:05,042 : INFO : 89 batches submitted to accumulate stats from 5696 documents (463000 virtual) 2018-03-28 00:37:05,167 : INFO : 90 batches submitted to accumulate stats from 5760 documents (468341 virtual) 2018-03-28 00:37:05,210 : INFO : 91 batches submitted to accumulate stats from 5824 documents (473805 virtual) 2018-03-28 00:37:05,254 : INFO : 92 batches submitted to accumulate stats from 5888 documents (479144 virtual) 2018-03-28 00:37:05,320 : INFO : 93 batches submitted to accumulate stats from 5952 documents (484277 virtual) 2018-03-28 00:37:05,390 : INFO : 94 batches submitted to accumulate stats from 6016 documents (490100 virtual) 2018-03-28 00:37:05,423 : INFO : 95 batches submitted to accumulate stats from 6080 documents (495729 virtual) 2018-03-28 00:37:05,514 : INFO : 96 batches submitted to accumulate stats from 6144 documents (500712 virtual) 2018-03-28 00:37:05,557 : INFO : 97 batches submitted to accumulate stats from 6208 documents (506253 virtual) 2018-03-28 00:37:05,617 : INFO : 98 batches submitted to accumulate stats from 6272 documents (511296 virtual) 2018-03-28 00:37:05,692 : INFO : 99 batches submitted to accumulate stats from 6336 documents (516045 virtual) 2018-03-28 00:37:05,749 : INFO : 100 batches submitted to accumulate stats from 6400 documents (521206 virtual) 2018-03-28 00:37:05,809 : INFO : 101 batches submitted to accumulate stats from 6464 documents (526237 virtual) 2018-03-28 00:37:05,851 : INFO : 102 batches submitted to accumulate stats from 6528 documents (531188 virtual) 2018-03-28 00:37:05,940 : INFO : 103 batches submitted to accumulate stats from 6592 documents (536374 virtual) 2018-03-28 00:37:05,974 : INFO : 104 batches submitted to accumulate stats from 6656 documents (541574 virtual) 2018-03-28 00:37:06,008 : INFO : 105 batches submitted to accumulate stats from 6720 documents (546747 virtual) 2018-03-28 00:37:06,118 : INFO : 106 batches submitted to accumulate stats from 6784 documents (551868 virtual) 2018-03-28 00:37:06,134 : INFO : 107 batches submitted to accumulate stats from 6848 documents (557264 virtual) 2018-03-28 00:37:06,193 : INFO : 108 batches submitted to accumulate stats from 6912 documents (562641 virtual) 2018-03-28 00:37:06,271 : INFO : 109 batches submitted to accumulate stats from 6976 documents (568059 virtual) 2018-03-28 00:37:06,284 : INFO : 110 batches submitted to accumulate stats from 7040 documents (573099 virtual) 2018-03-28 00:37:06,420 : INFO : 111 batches submitted to accumulate stats from 7104 documents (577546 virtual) 2018-03-28 00:37:06,427 : INFO : 112 batches submitted to accumulate stats from 7168 documents (582449 virtual) 2018-03-28 00:37:06,459 : INFO : 113 batches submitted to accumulate stats from 7232 documents (587997 virtual) 2018-03-28 00:37:06,582 : INFO : 114 batches submitted to accumulate stats from 7296 documents (592649 virtual) 2018-03-28 00:37:06,620 : INFO : 115 batches submitted to accumulate stats from 7360 documents (598365 virtual) 2018-03-28 00:37:06,630 : INFO : 116 batches submitted to accumulate stats from 7424 documents (603690 virtual) 2018-03-28 00:37:06,737 : INFO : 117 batches submitted to accumulate stats from 7488 documents (608587 virtual) 2018-03-28 00:37:06,791 : INFO : 118 batches submitted to accumulate stats from 7552 documents (613868 virtual) 2018-03-28 00:37:06,804 : INFO : 119 batches submitted to accumulate stats from 7616 documents (618819 virtual) 2018-03-28 00:37:06,926 : INFO : 120 batches submitted to accumulate stats from 7680 documents (624121 virtual) 2018-03-28 00:37:06,956 : INFO : 121 batches submitted to accumulate stats from 7744 documents (629269 virtual) 2018-03-28 00:37:06,985 : INFO : 122 batches submitted to accumulate stats from 7808 documents (635072 virtual) 2018-03-28 00:37:07,113 : INFO : 123 batches submitted to accumulate stats from 7872 documents (640050 virtual) 2018-03-28 00:37:07,136 : INFO : 124 batches submitted to accumulate stats from 7936 documents (645205 virtual) 2018-03-28 00:37:07,143 : INFO : 125 batches submitted to accumulate stats from 8000 documents (647463 virtual) 2018-03-28 00:37:07,442 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:07,468 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:07,448 : INFO : accumulator serialized 2018-03-28 00:37:07,526 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:07,531 : INFO : accumulator serialized 2018-03-28 00:37:07,476 : INFO : accumulator serialized 2018-03-28 00:37:07,596 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:37:07,680 : INFO : accumulated word occurrence stats for 648093 virtual documents 2018-03-28 00:37:07,951 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_25 2018-03-28 00:37:07,953 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-28 00:37:07,956 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:37:07,956 : INFO : setting ignored attribute state to None 2018-03-28 00:37:07,957 : INFO : setting ignored attribute id2word to None 2018-03-28 00:37:07,957 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_25 2018-03-28 00:37:07,958 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_25.state 2018-03-28 00:37:07,971 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_25.state 2018-03-28 00:37:07,986 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:37:08,032 : INFO : 1 batches submitted to accumulate stats from 64 documents (4982 virtual) 2018-03-28 00:37:08,039 : INFO : 2 batches submitted to accumulate stats from 128 documents (10157 virtual) 2018-03-28 00:37:08,047 : INFO : 3 batches submitted to accumulate stats from 192 documents (15057 virtual) 2018-03-28 00:37:08,054 : INFO : 4 batches submitted to accumulate stats from 256 documents (19941 virtual) 2018-03-28 00:37:08,060 : INFO : 5 batches submitted to accumulate stats from 320 documents (25048 virtual) 2018-03-28 00:37:08,067 : INFO : 6 batches submitted to accumulate stats from 384 documents (30581 virtual) 2018-03-28 00:37:08,225 : INFO : 7 batches submitted to accumulate stats from 448 documents (35845 virtual) 2018-03-28 00:37:08,253 : INFO : 8 batches submitted to accumulate stats from 512 documents (40777 virtual) 2018-03-28 00:37:08,261 : INFO : 9 batches submitted to accumulate stats from 576 documents (45638 virtual) 2018-03-28 00:37:08,407 : INFO : 10 batches submitted to accumulate stats from 640 documents (51118 virtual) 2018-03-28 00:37:08,442 : INFO : 11 batches submitted to accumulate stats from 704 documents (56407 virtual) 2018-03-28 00:37:08,485 : INFO : 12 batches submitted to accumulate stats from 768 documents (61056 virtual) 2018-03-28 00:37:08,605 : INFO : 13 batches submitted to accumulate stats from 832 documents (66787 virtual) 2018-03-28 00:37:08,614 : INFO : 14 batches submitted to accumulate stats from 896 documents (72403 virtual) 2018-03-28 00:37:08,690 : INFO : 15 batches submitted to accumulate stats from 960 documents (77560 virtual) 2018-03-28 00:37:08,797 : INFO : 16 batches submitted to accumulate stats from 1024 documents (83461 virtual) 2018-03-28 00:37:08,821 : INFO : 17 batches submitted to accumulate stats from 1088 documents (88623 virtual) 2018-03-28 00:37:08,870 : INFO : 18 batches submitted to accumulate stats from 1152 documents (93573 virtual) 2018-03-28 00:37:09,025 : INFO : 19 batches submitted to accumulate stats from 1216 documents (98302 virtual) 2018-03-28 00:37:09,037 : INFO : 20 batches submitted to accumulate stats from 1280 documents (104024 virtual) 2018-03-28 00:37:09,049 : INFO : 21 batches submitted to accumulate stats from 1344 documents (109331 virtual) 2018-03-28 00:37:09,220 : INFO : 22 batches submitted to accumulate stats from 1408 documents (114598 virtual) 2018-03-28 00:37:09,227 : INFO : 23 batches submitted to accumulate stats from 1472 documents (120281 virtual) 2018-03-28 00:37:09,266 : INFO : 24 batches submitted to accumulate stats from 1536 documents (125702 virtual) 2018-03-28 00:37:09,396 : INFO : 25 batches submitted to accumulate stats from 1600 documents (130886 virtual) 2018-03-28 00:37:09,442 : INFO : 26 batches submitted to accumulate stats from 1664 documents (135655 virtual) 2018-03-28 00:37:09,462 : INFO : 27 batches submitted to accumulate stats from 1728 documents (141221 virtual) 2018-03-28 00:37:09,591 : INFO : 28 batches submitted to accumulate stats from 1792 documents (146417 virtual) 2018-03-28 00:37:09,650 : INFO : 29 batches submitted to accumulate stats from 1856 documents (152022 virtual) 2018-03-28 00:37:09,667 : INFO : 30 batches submitted to accumulate stats from 1920 documents (157433 virtual) 2018-03-28 00:37:09,792 : INFO : 31 batches submitted to accumulate stats from 1984 documents (162621 virtual) 2018-03-28 00:37:09,826 : INFO : 32 batches submitted to accumulate stats from 2048 documents (167492 virtual) 2018-03-28 00:37:09,871 : INFO : 33 batches submitted to accumulate stats from 2112 documents (172332 virtual) 2018-03-28 00:37:10,003 : INFO : 34 batches submitted to accumulate stats from 2176 documents (177761 virtual) 2018-03-28 00:37:10,024 : INFO : 35 batches submitted to accumulate stats from 2240 documents (183357 virtual) 2018-03-28 00:37:10,064 : INFO : 36 batches submitted to accumulate stats from 2304 documents (188436 virtual) 2018-03-28 00:37:10,171 : INFO : 37 batches submitted to accumulate stats from 2368 documents (194151 virtual) 2018-03-28 00:37:10,241 : INFO : 38 batches submitted to accumulate stats from 2432 documents (200474 virtual) 2018-03-28 00:37:10,273 : INFO : 39 batches submitted to accumulate stats from 2496 documents (205373 virtual) 2018-03-28 00:37:10,357 : INFO : 40 batches submitted to accumulate stats from 2560 documents (210566 virtual) 2018-03-28 00:37:10,431 : INFO : 41 batches submitted to accumulate stats from 2624 documents (215073 virtual) 2018-03-28 00:37:10,479 : INFO : 42 batches submitted to accumulate stats from 2688 documents (220401 virtual) 2018-03-28 00:37:10,544 : INFO : 43 batches submitted to accumulate stats from 2752 documents (224827 virtual) 2018-03-28 00:37:10,663 : INFO : 44 batches submitted to accumulate stats from 2816 documents (229446 virtual) 2018-03-28 00:37:10,675 : INFO : 45 batches submitted to accumulate stats from 2880 documents (235053 virtual) 2018-03-28 00:37:10,715 : INFO : 46 batches submitted to accumulate stats from 2944 documents (239930 virtual) 2018-03-28 00:37:10,848 : INFO : 47 batches submitted to accumulate stats from 3008 documents (244676 virtual) 2018-03-28 00:37:10,874 : INFO : 48 batches submitted to accumulate stats from 3072 documents (249674 virtual) 2018-03-28 00:37:10,895 : INFO : 49 batches submitted to accumulate stats from 3136 documents (254623 virtual) 2018-03-28 00:37:11,000 : INFO : 50 batches submitted to accumulate stats from 3200 documents (260221 virtual) 2018-03-28 00:37:11,075 : INFO : 51 batches submitted to accumulate stats from 3264 documents (265277 virtual) 2018-03-28 00:37:11,091 : INFO : 52 batches submitted to accumulate stats from 3328 documents (270454 virtual) 2018-03-28 00:37:11,162 : INFO : 53 batches submitted to accumulate stats from 3392 documents (275654 virtual) 2018-03-28 00:37:11,285 : INFO : 54 batches submitted to accumulate stats from 3456 documents (280696 virtual) 2018-03-28 00:37:11,299 : INFO : 55 batches submitted to accumulate stats from 3520 documents (285590 virtual) 2018-03-28 00:37:11,350 : INFO : 56 batches submitted to accumulate stats from 3584 documents (290491 virtual) 2018-03-28 00:37:11,470 : INFO : 57 batches submitted to accumulate stats from 3648 documents (296198 virtual) 2018-03-28 00:37:11,493 : INFO : 58 batches submitted to accumulate stats from 3712 documents (301241 virtual) 2018-03-28 00:37:11,549 : INFO : 59 batches submitted to accumulate stats from 3776 documents (306370 virtual) 2018-03-28 00:37:11,637 : INFO : 60 batches submitted to accumulate stats from 3840 documents (311632 virtual) 2018-03-28 00:37:11,695 : INFO : 61 batches submitted to accumulate stats from 3904 documents (316831 virtual) 2018-03-28 00:37:11,726 : INFO : 62 batches submitted to accumulate stats from 3968 documents (321771 virtual) 2018-03-28 00:37:11,844 : INFO : 63 batches submitted to accumulate stats from 4032 documents (326505 virtual) 2018-03-28 00:37:11,858 : INFO : 64 batches submitted to accumulate stats from 4096 documents (331876 virtual) 2018-03-28 00:37:11,947 : INFO : 65 batches submitted to accumulate stats from 4160 documents (337046 virtual) 2018-03-28 00:37:12,049 : INFO : 66 batches submitted to accumulate stats from 4224 documents (342312 virtual) 2018-03-28 00:37:12,063 : INFO : 67 batches submitted to accumulate stats from 4288 documents (347404 virtual) 2018-03-28 00:37:12,136 : INFO : 68 batches submitted to accumulate stats from 4352 documents (352519 virtual) 2018-03-28 00:37:12,227 : INFO : 69 batches submitted to accumulate stats from 4416 documents (357694 virtual) 2018-03-28 00:37:12,280 : INFO : 70 batches submitted to accumulate stats from 4480 documents (362903 virtual) 2018-03-28 00:37:12,314 : INFO : 71 batches submitted to accumulate stats from 4544 documents (367575 virtual) 2018-03-28 00:37:12,430 : INFO : 72 batches submitted to accumulate stats from 4608 documents (372807 virtual) 2018-03-28 00:37:12,472 : INFO : 73 batches submitted to accumulate stats from 4672 documents (377997 virtual) 2018-03-28 00:37:12,500 : INFO : 74 batches submitted to accumulate stats from 4736 documents (383571 virtual) 2018-03-28 00:37:12,610 : INFO : 75 batches submitted to accumulate stats from 4800 documents (388804 virtual) 2018-03-28 00:37:12,668 : INFO : 76 batches submitted to accumulate stats from 4864 documents (393255 virtual) 2018-03-28 00:37:12,702 : INFO : 77 batches submitted to accumulate stats from 4928 documents (398540 virtual) 2018-03-28 00:37:12,824 : INFO : 78 batches submitted to accumulate stats from 4992 documents (403620 virtual) 2018-03-28 00:37:12,846 : INFO : 79 batches submitted to accumulate stats from 5056 documents (408677 virtual) 2018-03-28 00:37:12,917 : INFO : 80 batches submitted to accumulate stats from 5120 documents (414061 virtual) 2018-03-28 00:37:12,990 : INFO : 81 batches submitted to accumulate stats from 5184 documents (419596 virtual) 2018-03-28 00:37:13,034 : INFO : 82 batches submitted to accumulate stats from 5248 documents (424836 virtual) 2018-03-28 00:37:13,093 : INFO : 83 batches submitted to accumulate stats from 5312 documents (430133 virtual) 2018-03-28 00:37:13,202 : INFO : 84 batches submitted to accumulate stats from 5376 documents (435883 virtual) 2018-03-28 00:37:13,248 : INFO : 85 batches submitted to accumulate stats from 5440 documents (441434 virtual) 2018-03-28 00:37:13,281 : INFO : 86 batches submitted to accumulate stats from 5504 documents (447053 virtual) 2018-03-28 00:37:13,363 : INFO : 87 batches submitted to accumulate stats from 5568 documents (451735 virtual) 2018-03-28 00:37:13,472 : INFO : 88 batches submitted to accumulate stats from 5632 documents (457192 virtual) 2018-03-28 00:37:13,478 : INFO : 89 batches submitted to accumulate stats from 5696 documents (462355 virtual) 2018-03-28 00:37:13,561 : INFO : 90 batches submitted to accumulate stats from 5760 documents (467655 virtual) 2018-03-28 00:37:13,675 : INFO : 91 batches submitted to accumulate stats from 5824 documents (473056 virtual) 2018-03-28 00:37:13,706 : INFO : 92 batches submitted to accumulate stats from 5888 documents (478655 virtual) 2018-03-28 00:37:13,741 : INFO : 93 batches submitted to accumulate stats from 5952 documents (483934 virtual) 2018-03-28 00:37:13,877 : INFO : 94 batches submitted to accumulate stats from 6016 documents (489508 virtual) 2018-03-28 00:37:13,883 : INFO : 95 batches submitted to accumulate stats from 6080 documents (494980 virtual) 2018-03-28 00:37:13,934 : INFO : 96 batches submitted to accumulate stats from 6144 documents (500293 virtual) 2018-03-28 00:37:14,086 : INFO : 97 batches submitted to accumulate stats from 6208 documents (505658 virtual) 2018-03-28 00:37:14,095 : INFO : 98 batches submitted to accumulate stats from 6272 documents (510711 virtual) 2018-03-28 00:37:14,142 : INFO : 99 batches submitted to accumulate stats from 6336 documents (515435 virtual) 2018-03-28 00:37:14,283 : INFO : 100 batches submitted to accumulate stats from 6400 documents (520602 virtual) 2018-03-28 00:37:14,314 : INFO : 101 batches submitted to accumulate stats from 6464 documents (525661 virtual) 2018-03-28 00:37:14,342 : INFO : 102 batches submitted to accumulate stats from 6528 documents (530650 virtual) 2018-03-28 00:37:14,484 : INFO : 103 batches submitted to accumulate stats from 6592 documents (535725 virtual) 2018-03-28 00:37:14,511 : INFO : 104 batches submitted to accumulate stats from 6656 documents (540936 virtual) 2018-03-28 00:37:14,518 : INFO : 105 batches submitted to accumulate stats from 6720 documents (546058 virtual) 2018-03-28 00:37:14,674 : INFO : 106 batches submitted to accumulate stats from 6784 documents (551013 virtual) 2018-03-28 00:37:14,690 : INFO : 107 batches submitted to accumulate stats from 6848 documents (556507 virtual) 2018-03-28 00:37:14,742 : INFO : 108 batches submitted to accumulate stats from 6912 documents (561629 virtual) 2018-03-28 00:37:14,908 : INFO : 109 batches submitted to accumulate stats from 6976 documents (567108 virtual) 2018-03-28 00:37:14,917 : INFO : 110 batches submitted to accumulate stats from 7040 documents (572121 virtual) 2018-03-28 00:37:14,959 : INFO : 111 batches submitted to accumulate stats from 7104 documents (576719 virtual) 2018-03-28 00:37:15,096 : INFO : 112 batches submitted to accumulate stats from 7168 documents (581446 virtual) 2018-03-28 00:37:15,113 : INFO : 113 batches submitted to accumulate stats from 7232 documents (587049 virtual) 2018-03-28 00:37:15,156 : INFO : 114 batches submitted to accumulate stats from 7296 documents (591835 virtual) 2018-03-28 00:37:15,289 : INFO : 115 batches submitted to accumulate stats from 7360 documents (597115 virtual) 2018-03-28 00:37:15,312 : INFO : 116 batches submitted to accumulate stats from 7424 documents (602688 virtual) 2018-03-28 00:37:15,337 : INFO : 117 batches submitted to accumulate stats from 7488 documents (607592 virtual) 2018-03-28 00:37:15,485 : INFO : 118 batches submitted to accumulate stats from 7552 documents (612613 virtual) 2018-03-28 00:37:15,496 : INFO : 119 batches submitted to accumulate stats from 7616 documents (617826 virtual) 2018-03-28 00:37:15,527 : INFO : 120 batches submitted to accumulate stats from 7680 documents (623064 virtual) 2018-03-28 00:37:15,665 : INFO : 121 batches submitted to accumulate stats from 7744 documents (628311 virtual) 2018-03-28 00:37:15,716 : INFO : 122 batches submitted to accumulate stats from 7808 documents (634180 virtual) 2018-03-28 00:37:15,727 : INFO : 123 batches submitted to accumulate stats from 7872 documents (639156 virtual) 2018-03-28 00:37:15,833 : INFO : 124 batches submitted to accumulate stats from 7936 documents (644213 virtual) 2018-03-28 00:37:15,903 : INFO : 125 batches submitted to accumulate stats from 8000 documents (647458 virtual) 2018-03-28 00:37:16,116 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:16,123 : INFO : accumulator serialized 2018-03-28 00:37:16,238 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:16,244 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:16,243 : INFO : accumulator serialized 2018-03-28 00:37:16,249 : INFO : accumulator serialized 2018-03-28 00:37:16,355 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:37:16,491 : INFO : accumulated word occurrence stats for 648117 virtual documents 2018-03-28 00:37:16,823 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_30 2018-03-28 00:37:16,824 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-28 00:37:16,828 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:37:16,829 : INFO : setting ignored attribute state to None 2018-03-28 00:37:16,829 : INFO : setting ignored attribute id2word to None 2018-03-28 00:37:16,830 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_30 2018-03-28 00:37:16,830 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_30.state 2018-03-28 00:37:16,845 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_30.state 2018-03-28 00:37:16,863 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:37:16,910 : INFO : 1 batches submitted to accumulate stats from 64 documents (4982 virtual) 2018-03-28 00:37:16,917 : INFO : 2 batches submitted to accumulate stats from 128 documents (10157 virtual) 2018-03-28 00:37:16,924 : INFO : 3 batches submitted to accumulate stats from 192 documents (15057 virtual) 2018-03-28 00:37:16,932 : INFO : 4 batches submitted to accumulate stats from 256 documents (19941 virtual) 2018-03-28 00:37:16,939 : INFO : 5 batches submitted to accumulate stats from 320 documents (25048 virtual) 2018-03-28 00:37:16,946 : INFO : 6 batches submitted to accumulate stats from 384 documents (30581 virtual) 2018-03-28 00:37:17,108 : INFO : 7 batches submitted to accumulate stats from 448 documents (35845 virtual) 2018-03-28 00:37:17,136 : INFO : 8 batches submitted to accumulate stats from 512 documents (40777 virtual) 2018-03-28 00:37:17,159 : INFO : 9 batches submitted to accumulate stats from 576 documents (45638 virtual) 2018-03-28 00:37:17,356 : INFO : 10 batches submitted to accumulate stats from 640 documents (51118 virtual) 2018-03-28 00:37:17,363 : INFO : 11 batches submitted to accumulate stats from 704 documents (56333 virtual) 2018-03-28 00:37:17,379 : INFO : 12 batches submitted to accumulate stats from 768 documents (60884 virtual) 2018-03-28 00:37:17,554 : INFO : 13 batches submitted to accumulate stats from 832 documents (66653 virtual) 2018-03-28 00:37:17,573 : INFO : 14 batches submitted to accumulate stats from 896 documents (72104 virtual) 2018-03-28 00:37:17,590 : INFO : 15 batches submitted to accumulate stats from 960 documents (77395 virtual) 2018-03-28 00:37:17,758 : INFO : 16 batches submitted to accumulate stats from 1024 documents (83476 virtual) 2018-03-28 00:37:17,767 : INFO : 17 batches submitted to accumulate stats from 1088 documents (88506 virtual) 2018-03-28 00:37:17,789 : INFO : 18 batches submitted to accumulate stats from 1152 documents (93382 virtual) 2018-03-28 00:37:17,966 : INFO : 19 batches submitted to accumulate stats from 1216 documents (98195 virtual) 2018-03-28 00:37:17,978 : INFO : 20 batches submitted to accumulate stats from 1280 documents (103859 virtual) 2018-03-28 00:37:17,994 : INFO : 21 batches submitted to accumulate stats from 1344 documents (109190 virtual) 2018-03-28 00:37:18,162 : INFO : 22 batches submitted to accumulate stats from 1408 documents (114487 virtual) 2018-03-28 00:37:18,177 : INFO : 23 batches submitted to accumulate stats from 1472 documents (120192 virtual) 2018-03-28 00:37:18,221 : INFO : 24 batches submitted to accumulate stats from 1536 documents (125318 virtual) 2018-03-28 00:37:18,338 : INFO : 25 batches submitted to accumulate stats from 1600 documents (130728 virtual) 2018-03-28 00:37:18,404 : INFO : 26 batches submitted to accumulate stats from 1664 documents (135604 virtual) 2018-03-28 00:37:18,423 : INFO : 27 batches submitted to accumulate stats from 1728 documents (141107 virtual) 2018-03-28 00:37:18,553 : INFO : 28 batches submitted to accumulate stats from 1792 documents (146233 virtual) 2018-03-28 00:37:18,605 : INFO : 29 batches submitted to accumulate stats from 1856 documents (151723 virtual) 2018-03-28 00:37:18,646 : INFO : 30 batches submitted to accumulate stats from 1920 documents (157313 virtual) 2018-03-28 00:37:18,760 : INFO : 31 batches submitted to accumulate stats from 1984 documents (162270 virtual) 2018-03-28 00:37:18,766 : INFO : 32 batches submitted to accumulate stats from 2048 documents (167221 virtual) 2018-03-28 00:37:18,876 : INFO : 33 batches submitted to accumulate stats from 2112 documents (172027 virtual) 2018-03-28 00:37:18,960 : INFO : 34 batches submitted to accumulate stats from 2176 documents (177615 virtual) 2018-03-28 00:37:18,980 : INFO : 35 batches submitted to accumulate stats from 2240 documents (183248 virtual) 2018-03-28 00:37:19,082 : INFO : 36 batches submitted to accumulate stats from 2304 documents (188186 virtual) 2018-03-28 00:37:19,159 : INFO : 37 batches submitted to accumulate stats from 2368 documents (193893 virtual) 2018-03-28 00:37:19,166 : INFO : 38 batches submitted to accumulate stats from 2432 documents (200219 virtual) 2018-03-28 00:37:19,252 : INFO : 39 batches submitted to accumulate stats from 2496 documents (205228 virtual) 2018-03-28 00:37:19,369 : INFO : 40 batches submitted to accumulate stats from 2560 documents (210271 virtual) 2018-03-28 00:37:19,379 : INFO : 41 batches submitted to accumulate stats from 2624 documents (214964 virtual) 2018-03-28 00:37:19,456 : INFO : 42 batches submitted to accumulate stats from 2688 documents (220057 virtual) 2018-03-28 00:37:19,591 : INFO : 43 batches submitted to accumulate stats from 2752 documents (224751 virtual) 2018-03-28 00:37:19,611 : INFO : 44 batches submitted to accumulate stats from 2816 documents (229295 virtual) 2018-03-28 00:37:19,643 : INFO : 45 batches submitted to accumulate stats from 2880 documents (234871 virtual) 2018-03-28 00:37:19,782 : INFO : 46 batches submitted to accumulate stats from 2944 documents (239789 virtual) 2018-03-28 00:37:19,788 : INFO : 47 batches submitted to accumulate stats from 3008 documents (244548 virtual) 2018-03-28 00:37:19,855 : INFO : 48 batches submitted to accumulate stats from 3072 documents (249553 virtual) 2018-03-28 00:37:19,945 : INFO : 49 batches submitted to accumulate stats from 3136 documents (254401 virtual) 2018-03-28 00:37:19,985 : INFO : 50 batches submitted to accumulate stats from 3200 documents (260033 virtual) 2018-03-28 00:37:20,090 : INFO : 51 batches submitted to accumulate stats from 3264 documents (265007 virtual) 2018-03-28 00:37:20,122 : INFO : 52 batches submitted to accumulate stats from 3328 documents (270346 virtual) 2018-03-28 00:37:20,174 : INFO : 53 batches submitted to accumulate stats from 3392 documents (275486 virtual) 2018-03-28 00:37:20,266 : INFO : 54 batches submitted to accumulate stats from 3456 documents (280523 virtual) 2018-03-28 00:37:20,332 : INFO : 55 batches submitted to accumulate stats from 3520 documents (285462 virtual) 2018-03-28 00:37:20,378 : INFO : 56 batches submitted to accumulate stats from 3584 documents (290399 virtual) 2018-03-28 00:37:20,452 : INFO : 57 batches submitted to accumulate stats from 3648 documents (295784 virtual) 2018-03-28 00:37:20,544 : INFO : 58 batches submitted to accumulate stats from 3712 documents (301120 virtual) 2018-03-28 00:37:20,594 : INFO : 59 batches submitted to accumulate stats from 3776 documents (306316 virtual) 2018-03-28 00:37:20,634 : INFO : 60 batches submitted to accumulate stats from 3840 documents (311555 virtual) 2018-03-28 00:37:20,727 : INFO : 61 batches submitted to accumulate stats from 3904 documents (316771 virtual) 2018-03-28 00:37:20,793 : INFO : 62 batches submitted to accumulate stats from 3968 documents (321597 virtual) 2018-03-28 00:37:20,847 : INFO : 63 batches submitted to accumulate stats from 4032 documents (326307 virtual) 2018-03-28 00:37:20,947 : INFO : 64 batches submitted to accumulate stats from 4096 documents (331751 virtual) 2018-03-28 00:37:20,985 : INFO : 65 batches submitted to accumulate stats from 4160 documents (336932 virtual) 2018-03-28 00:37:21,050 : INFO : 66 batches submitted to accumulate stats from 4224 documents (342094 virtual) 2018-03-28 00:37:21,168 : INFO : 67 batches submitted to accumulate stats from 4288 documents (347187 virtual) 2018-03-28 00:37:21,177 : INFO : 68 batches submitted to accumulate stats from 4352 documents (352221 virtual) 2018-03-28 00:37:21,224 : INFO : 69 batches submitted to accumulate stats from 4416 documents (357294 virtual) 2018-03-28 00:37:21,373 : INFO : 70 batches submitted to accumulate stats from 4480 documents (362576 virtual) 2018-03-28 00:37:21,386 : INFO : 71 batches submitted to accumulate stats from 4544 documents (367317 virtual) 2018-03-28 00:37:21,449 : INFO : 72 batches submitted to accumulate stats from 4608 documents (372593 virtual) 2018-03-28 00:37:21,538 : INFO : 73 batches submitted to accumulate stats from 4672 documents (377717 virtual) 2018-03-28 00:37:21,586 : INFO : 74 batches submitted to accumulate stats from 4736 documents (383450 virtual) 2018-03-28 00:37:21,645 : INFO : 75 batches submitted to accumulate stats from 4800 documents (388532 virtual) 2018-03-28 00:37:21,759 : INFO : 76 batches submitted to accumulate stats from 4864 documents (393133 virtual) 2018-03-28 00:37:21,775 : INFO : 77 batches submitted to accumulate stats from 4928 documents (398349 virtual) 2018-03-28 00:37:21,833 : INFO : 78 batches submitted to accumulate stats from 4992 documents (403438 virtual) 2018-03-28 00:37:21,969 : INFO : 79 batches submitted to accumulate stats from 5056 documents (408497 virtual) 2018-03-28 00:37:21,984 : INFO : 80 batches submitted to accumulate stats from 5120 documents (413883 virtual) 2018-03-28 00:37:22,021 : INFO : 81 batches submitted to accumulate stats from 5184 documents (419375 virtual) 2018-03-28 00:37:22,165 : INFO : 82 batches submitted to accumulate stats from 5248 documents (424667 virtual) 2018-03-28 00:37:22,193 : INFO : 83 batches submitted to accumulate stats from 5312 documents (429889 virtual) 2018-03-28 00:37:22,200 : INFO : 84 batches submitted to accumulate stats from 5376 documents (435838 virtual) 2018-03-28 00:37:22,397 : INFO : 85 batches submitted to accumulate stats from 5440 documents (441360 virtual) 2018-03-28 00:37:22,427 : INFO : 86 batches submitted to accumulate stats from 5504 documents (447116 virtual) 2018-03-28 00:37:22,446 : INFO : 87 batches submitted to accumulate stats from 5568 documents (451645 virtual) 2018-03-28 00:37:22,615 : INFO : 88 batches submitted to accumulate stats from 5632 documents (457033 virtual) 2018-03-28 00:37:22,637 : INFO : 89 batches submitted to accumulate stats from 5696 documents (462287 virtual) 2018-03-28 00:37:22,656 : INFO : 90 batches submitted to accumulate stats from 5760 documents (467656 virtual) 2018-03-28 00:37:22,821 : INFO : 91 batches submitted to accumulate stats from 5824 documents (472972 virtual) 2018-03-28 00:37:22,831 : INFO : 92 batches submitted to accumulate stats from 5888 documents (478670 virtual) 2018-03-28 00:37:22,871 : INFO : 93 batches submitted to accumulate stats from 5952 documents (483600 virtual) 2018-03-28 00:37:23,018 : INFO : 94 batches submitted to accumulate stats from 6016 documents (489170 virtual) 2018-03-28 00:37:23,036 : INFO : 95 batches submitted to accumulate stats from 6080 documents (494885 virtual) 2018-03-28 00:37:23,083 : INFO : 96 batches submitted to accumulate stats from 6144 documents (500078 virtual) 2018-03-28 00:37:23,233 : INFO : 97 batches submitted to accumulate stats from 6208 documents (505384 virtual) 2018-03-28 00:37:23,256 : INFO : 98 batches submitted to accumulate stats from 6272 documents (510422 virtual) 2018-03-28 00:37:23,280 : INFO : 99 batches submitted to accumulate stats from 6336 documents (515440 virtual) 2018-03-28 00:37:23,452 : INFO : 100 batches submitted to accumulate stats from 6400 documents (520316 virtual) 2018-03-28 00:37:23,466 : INFO : 101 batches submitted to accumulate stats from 6464 documents (525437 virtual) 2018-03-28 00:37:23,472 : INFO : 102 batches submitted to accumulate stats from 6528 documents (530499 virtual) 2018-03-28 00:37:23,647 : INFO : 103 batches submitted to accumulate stats from 6592 documents (535463 virtual) 2018-03-28 00:37:23,680 : INFO : 104 batches submitted to accumulate stats from 6656 documents (540758 virtual) 2018-03-28 00:37:23,692 : INFO : 105 batches submitted to accumulate stats from 6720 documents (546012 virtual) 2018-03-28 00:37:23,835 : INFO : 106 batches submitted to accumulate stats from 6784 documents (550852 virtual) 2018-03-28 00:37:23,867 : INFO : 107 batches submitted to accumulate stats from 6848 documents (556235 virtual) 2018-03-28 00:37:23,884 : INFO : 108 batches submitted to accumulate stats from 6912 documents (561393 virtual) 2018-03-28 00:37:24,053 : INFO : 109 batches submitted to accumulate stats from 6976 documents (566997 virtual) 2018-03-28 00:37:24,059 : INFO : 110 batches submitted to accumulate stats from 7040 documents (571906 virtual) 2018-03-28 00:37:24,097 : INFO : 111 batches submitted to accumulate stats from 7104 documents (576643 virtual) 2018-03-28 00:37:24,233 : INFO : 112 batches submitted to accumulate stats from 7168 documents (581432 virtual) 2018-03-28 00:37:24,253 : INFO : 113 batches submitted to accumulate stats from 7232 documents (586908 virtual) 2018-03-28 00:37:24,312 : INFO : 114 batches submitted to accumulate stats from 7296 documents (591452 virtual) 2018-03-28 00:37:24,435 : INFO : 115 batches submitted to accumulate stats from 7360 documents (596898 virtual) 2018-03-28 00:37:24,471 : INFO : 116 batches submitted to accumulate stats from 7424 documents (602443 virtual) 2018-03-28 00:37:24,485 : INFO : 117 batches submitted to accumulate stats from 7488 documents (607306 virtual) 2018-03-28 00:37:24,606 : INFO : 118 batches submitted to accumulate stats from 7552 documents (612546 virtual) 2018-03-28 00:37:24,672 : INFO : 119 batches submitted to accumulate stats from 7616 documents (617634 virtual) 2018-03-28 00:37:24,692 : INFO : 120 batches submitted to accumulate stats from 7680 documents (622816 virtual) 2018-03-28 00:37:24,831 : INFO : 121 batches submitted to accumulate stats from 7744 documents (627780 virtual) 2018-03-28 00:37:24,878 : INFO : 122 batches submitted to accumulate stats from 7808 documents (633874 virtual) 2018-03-28 00:37:24,895 : INFO : 123 batches submitted to accumulate stats from 7872 documents (638920 virtual) 2018-03-28 00:37:25,046 : INFO : 124 batches submitted to accumulate stats from 7936 documents (643953 virtual) 2018-03-28 00:37:25,053 : INFO : 125 batches submitted to accumulate stats from 8000 documents (647636 virtual) 2018-03-28 00:37:25,315 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:25,410 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:25,413 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:25,320 : INFO : accumulator serialized 2018-03-28 00:37:25,417 : INFO : accumulator serialized 2018-03-28 00:37:25,423 : INFO : accumulator serialized 2018-03-28 00:37:25,555 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:37:25,721 : INFO : accumulated word occurrence stats for 648298 virtual documents 2018-03-28 00:37:26,128 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_35 2018-03-28 00:37:26,129 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_35.expElogbeta.npy with mmap=None 2018-03-28 00:37:26,133 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:37:26,134 : INFO : setting ignored attribute state to None 2018-03-28 00:37:26,134 : INFO : setting ignored attribute id2word to None 2018-03-28 00:37:26,135 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_35 2018-03-28 00:37:26,135 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_35.state 2018-03-28 00:37:26,153 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_35.state 2018-03-28 00:37:26,171 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:37:26,218 : INFO : 1 batches submitted to accumulate stats from 64 documents (4982 virtual) 2018-03-28 00:37:26,226 : INFO : 2 batches submitted to accumulate stats from 128 documents (10157 virtual) 2018-03-28 00:37:26,233 : INFO : 3 batches submitted to accumulate stats from 192 documents (15057 virtual) 2018-03-28 00:37:26,240 : INFO : 4 batches submitted to accumulate stats from 256 documents (19941 virtual) 2018-03-28 00:37:26,248 : INFO : 5 batches submitted to accumulate stats from 320 documents (25048 virtual) 2018-03-28 00:37:26,255 : INFO : 6 batches submitted to accumulate stats from 384 documents (30581 virtual) 2018-03-28 00:37:26,433 : INFO : 7 batches submitted to accumulate stats from 448 documents (35845 virtual) 2018-03-28 00:37:26,455 : INFO : 8 batches submitted to accumulate stats from 512 documents (40777 virtual) 2018-03-28 00:37:26,481 : INFO : 9 batches submitted to accumulate stats from 576 documents (45638 virtual) 2018-03-28 00:37:26,636 : INFO : 10 batches submitted to accumulate stats from 640 documents (51118 virtual) 2018-03-28 00:37:26,682 : INFO : 11 batches submitted to accumulate stats from 704 documents (56333 virtual) 2018-03-28 00:37:26,689 : INFO : 12 batches submitted to accumulate stats from 768 documents (60983 virtual) 2018-03-28 00:37:26,857 : INFO : 13 batches submitted to accumulate stats from 832 documents (66662 virtual) 2018-03-28 00:37:26,880 : INFO : 14 batches submitted to accumulate stats from 896 documents (72096 virtual) 2018-03-28 00:37:26,913 : INFO : 15 batches submitted to accumulate stats from 960 documents (77271 virtual) 2018-03-28 00:37:27,059 : INFO : 16 batches submitted to accumulate stats from 1024 documents (83303 virtual) 2018-03-28 00:37:27,094 : INFO : 17 batches submitted to accumulate stats from 1088 documents (88427 virtual) 2018-03-28 00:37:27,139 : INFO : 18 batches submitted to accumulate stats from 1152 documents (93247 virtual) 2018-03-28 00:37:27,281 : INFO : 19 batches submitted to accumulate stats from 1216 documents (98043 virtual) 2018-03-28 00:37:27,288 : INFO : 20 batches submitted to accumulate stats from 1280 documents (103776 virtual) 2018-03-28 00:37:27,389 : INFO : 21 batches submitted to accumulate stats from 1344 documents (109033 virtual) 2018-03-28 00:37:27,471 : INFO : 22 batches submitted to accumulate stats from 1408 documents (114396 virtual) 2018-03-28 00:37:27,529 : INFO : 23 batches submitted to accumulate stats from 1472 documents (120092 virtual) 2018-03-28 00:37:27,598 : INFO : 24 batches submitted to accumulate stats from 1536 documents (125140 virtual) 2018-03-28 00:37:27,669 : INFO : 25 batches submitted to accumulate stats from 1600 documents (130666 virtual) 2018-03-28 00:37:27,749 : INFO : 26 batches submitted to accumulate stats from 1664 documents (135545 virtual) 2018-03-28 00:37:27,825 : INFO : 27 batches submitted to accumulate stats from 1728 documents (140960 virtual) 2018-03-28 00:37:27,910 : INFO : 28 batches submitted to accumulate stats from 1792 documents (146130 virtual) 2018-03-28 00:37:27,951 : INFO : 29 batches submitted to accumulate stats from 1856 documents (151717 virtual) 2018-03-28 00:37:28,053 : INFO : 30 batches submitted to accumulate stats from 1920 documents (157307 virtual) 2018-03-28 00:37:28,134 : INFO : 31 batches submitted to accumulate stats from 1984 documents (162264 virtual) 2018-03-28 00:37:28,140 : INFO : 32 batches submitted to accumulate stats from 2048 documents (167215 virtual) 2018-03-28 00:37:28,301 : INFO : 33 batches submitted to accumulate stats from 2112 documents (172021 virtual) 2018-03-28 00:37:28,334 : INFO : 34 batches submitted to accumulate stats from 2176 documents (177510 virtual) 2018-03-28 00:37:28,350 : INFO : 35 batches submitted to accumulate stats from 2240 documents (183041 virtual) 2018-03-28 00:37:28,535 : INFO : 36 batches submitted to accumulate stats from 2304 documents (188101 virtual) 2018-03-28 00:37:28,546 : INFO : 37 batches submitted to accumulate stats from 2368 documents (193768 virtual) 2018-03-28 00:37:28,565 : INFO : 38 batches submitted to accumulate stats from 2432 documents (200069 virtual) 2018-03-28 00:37:28,719 : INFO : 39 batches submitted to accumulate stats from 2496 documents (205101 virtual) 2018-03-28 00:37:28,770 : INFO : 40 batches submitted to accumulate stats from 2560 documents (210117 virtual) 2018-03-28 00:37:28,785 : INFO : 41 batches submitted to accumulate stats from 2624 documents (214878 virtual) 2018-03-28 00:37:28,958 : INFO : 42 batches submitted to accumulate stats from 2688 documents (220029 virtual) 2018-03-28 00:37:28,985 : INFO : 43 batches submitted to accumulate stats from 2752 documents (224625 virtual) 2018-03-28 00:37:29,034 : INFO : 44 batches submitted to accumulate stats from 2816 documents (229171 virtual) 2018-03-28 00:37:29,167 : INFO : 45 batches submitted to accumulate stats from 2880 documents (234770 virtual) 2018-03-28 00:37:29,184 : INFO : 46 batches submitted to accumulate stats from 2944 documents (239643 virtual) 2018-03-28 00:37:29,230 : INFO : 47 batches submitted to accumulate stats from 3008 documents (244507 virtual) 2018-03-28 00:37:29,355 : INFO : 48 batches submitted to accumulate stats from 3072 documents (249507 virtual) 2018-03-28 00:37:29,371 : INFO : 49 batches submitted to accumulate stats from 3136 documents (254338 virtual) 2018-03-28 00:37:29,461 : INFO : 50 batches submitted to accumulate stats from 3200 documents (259842 virtual) 2018-03-28 00:37:29,573 : INFO : 51 batches submitted to accumulate stats from 3264 documents (264839 virtual) 2018-03-28 00:37:29,595 : INFO : 52 batches submitted to accumulate stats from 3328 documents (270105 virtual) 2018-03-28 00:37:29,663 : INFO : 53 batches submitted to accumulate stats from 3392 documents (275383 virtual) 2018-03-28 00:37:29,814 : INFO : 54 batches submitted to accumulate stats from 3456 documents (280287 virtual) 2018-03-28 00:37:29,820 : INFO : 55 batches submitted to accumulate stats from 3520 documents (285341 virtual) 2018-03-28 00:37:29,906 : INFO : 56 batches submitted to accumulate stats from 3584 documents (290319 virtual) 2018-03-28 00:37:30,022 : INFO : 57 batches submitted to accumulate stats from 3648 documents (295502 virtual) 2018-03-28 00:37:30,043 : INFO : 58 batches submitted to accumulate stats from 3712 documents (300916 virtual) 2018-03-28 00:37:30,128 : INFO : 59 batches submitted to accumulate stats from 3776 documents (306009 virtual) 2018-03-28 00:37:30,230 : INFO : 60 batches submitted to accumulate stats from 3840 documents (311303 virtual) 2018-03-28 00:37:30,236 : INFO : 61 batches submitted to accumulate stats from 3904 documents (316424 virtual) 2018-03-28 00:37:30,329 : INFO : 62 batches submitted to accumulate stats from 3968 documents (321297 virtual) 2018-03-28 00:37:30,424 : INFO : 63 batches submitted to accumulate stats from 4032 documents (325989 virtual) 2018-03-28 00:37:30,492 : INFO : 64 batches submitted to accumulate stats from 4096 documents (331472 virtual) 2018-03-28 00:37:30,527 : INFO : 65 batches submitted to accumulate stats from 4160 documents (336773 virtual) 2018-03-28 00:37:30,648 : INFO : 66 batches submitted to accumulate stats from 4224 documents (341920 virtual) 2018-03-28 00:37:30,712 : INFO : 67 batches submitted to accumulate stats from 4288 documents (347006 virtual) 2018-03-28 00:37:30,731 : INFO : 68 batches submitted to accumulate stats from 4352 documents (351876 virtual) 2018-03-28 00:37:30,866 : INFO : 69 batches submitted to accumulate stats from 4416 documents (356999 virtual) 2018-03-28 00:37:30,915 : INFO : 70 batches submitted to accumulate stats from 4480 documents (362347 virtual) 2018-03-28 00:37:30,946 : INFO : 71 batches submitted to accumulate stats from 4544 documents (367042 virtual) 2018-03-28 00:37:31,082 : INFO : 72 batches submitted to accumulate stats from 4608 documents (372395 virtual) 2018-03-28 00:37:31,125 : INFO : 73 batches submitted to accumulate stats from 4672 documents (377514 virtual) 2018-03-28 00:37:31,152 : INFO : 74 batches submitted to accumulate stats from 4736 documents (383113 virtual) 2018-03-28 00:37:31,288 : INFO : 75 batches submitted to accumulate stats from 4800 documents (388117 virtual) 2018-03-28 00:37:31,331 : INFO : 76 batches submitted to accumulate stats from 4864 documents (392844 virtual) 2018-03-28 00:37:31,347 : INFO : 77 batches submitted to accumulate stats from 4928 documents (398082 virtual) 2018-03-28 00:37:31,523 : INFO : 78 batches submitted to accumulate stats from 4992 documents (403199 virtual) 2018-03-28 00:37:31,549 : INFO : 79 batches submitted to accumulate stats from 5056 documents (408222 virtual) 2018-03-28 00:37:31,566 : INFO : 80 batches submitted to accumulate stats from 5120 documents (413552 virtual) 2018-03-28 00:37:31,741 : INFO : 81 batches submitted to accumulate stats from 5184 documents (419000 virtual) 2018-03-28 00:37:31,751 : INFO : 82 batches submitted to accumulate stats from 5248 documents (424334 virtual) 2018-03-28 00:37:31,760 : INFO : 83 batches submitted to accumulate stats from 5312 documents (429623 virtual) 2018-03-28 00:37:31,916 : INFO : 84 batches submitted to accumulate stats from 5376 documents (435549 virtual) 2018-03-28 00:37:31,963 : INFO : 85 batches submitted to accumulate stats from 5440 documents (441018 virtual) 2018-03-28 00:37:32,006 : INFO : 86 batches submitted to accumulate stats from 5504 documents (446736 virtual) 2018-03-28 00:37:32,136 : INFO : 87 batches submitted to accumulate stats from 5568 documents (451230 virtual) 2018-03-28 00:37:32,197 : INFO : 88 batches submitted to accumulate stats from 5632 documents (456628 virtual) 2018-03-28 00:37:32,218 : INFO : 89 batches submitted to accumulate stats from 5696 documents (461820 virtual) 2018-03-28 00:37:32,419 : INFO : 90 batches submitted to accumulate stats from 5760 documents (467303 virtual) 2018-03-28 00:37:32,465 : INFO : 91 batches submitted to accumulate stats from 5824 documents (472558 virtual) 2018-03-28 00:37:32,473 : INFO : 92 batches submitted to accumulate stats from 5888 documents (478313 virtual) 2018-03-28 00:37:32,618 : INFO : 93 batches submitted to accumulate stats from 5952 documents (483414 virtual) 2018-03-28 00:37:32,673 : INFO : 94 batches submitted to accumulate stats from 6016 documents (488906 virtual) 2018-03-28 00:37:32,679 : INFO : 95 batches submitted to accumulate stats from 6080 documents (494363 virtual) 2018-03-28 00:37:32,834 : INFO : 96 batches submitted to accumulate stats from 6144 documents (499815 virtual) 2018-03-28 00:37:32,890 : INFO : 97 batches submitted to accumulate stats from 6208 documents (505068 virtual) 2018-03-28 00:37:32,923 : INFO : 98 batches submitted to accumulate stats from 6272 documents (510055 virtual) 2018-03-28 00:37:33,024 : INFO : 99 batches submitted to accumulate stats from 6336 documents (515019 virtual) 2018-03-28 00:37:33,140 : INFO : 100 batches submitted to accumulate stats from 6400 documents (520003 virtual) 2018-03-28 00:37:33,157 : INFO : 101 batches submitted to accumulate stats from 6464 documents (525265 virtual) 2018-03-28 00:37:33,253 : INFO : 102 batches submitted to accumulate stats from 6528 documents (530099 virtual) 2018-03-28 00:37:33,359 : INFO : 103 batches submitted to accumulate stats from 6592 documents (535097 virtual) 2018-03-28 00:37:33,366 : INFO : 104 batches submitted to accumulate stats from 6656 documents (540175 virtual) 2018-03-28 00:37:33,452 : INFO : 105 batches submitted to accumulate stats from 6720 documents (545797 virtual) 2018-03-28 00:37:33,577 : INFO : 106 batches submitted to accumulate stats from 6784 documents (550508 virtual) 2018-03-28 00:37:33,605 : INFO : 107 batches submitted to accumulate stats from 6848 documents (555898 virtual) 2018-03-28 00:37:33,632 : INFO : 108 batches submitted to accumulate stats from 6912 documents (560666 virtual) 2018-03-28 00:37:33,792 : INFO : 109 batches submitted to accumulate stats from 6976 documents (566668 virtual) 2018-03-28 00:37:33,801 : INFO : 110 batches submitted to accumulate stats from 7040 documents (571469 virtual) 2018-03-28 00:37:33,860 : INFO : 111 batches submitted to accumulate stats from 7104 documents (576162 virtual) 2018-03-28 00:37:33,980 : INFO : 112 batches submitted to accumulate stats from 7168 documents (580837 virtual) 2018-03-28 00:37:34,030 : INFO : 113 batches submitted to accumulate stats from 7232 documents (586464 virtual) 2018-03-28 00:37:34,076 : INFO : 114 batches submitted to accumulate stats from 7296 documents (590878 virtual) 2018-03-28 00:37:34,209 : INFO : 115 batches submitted to accumulate stats from 7360 documents (596510 virtual) 2018-03-28 00:37:34,216 : INFO : 116 batches submitted to accumulate stats from 7424 documents (602165 virtual) 2018-03-28 00:37:34,298 : INFO : 117 batches submitted to accumulate stats from 7488 documents (607054 virtual) 2018-03-28 00:37:34,421 : INFO : 118 batches submitted to accumulate stats from 7552 documents (612133 virtual) 2018-03-28 00:37:34,430 : INFO : 119 batches submitted to accumulate stats from 7616 documents (617179 virtual) 2018-03-28 00:37:34,486 : INFO : 120 batches submitted to accumulate stats from 7680 documents (622297 virtual) 2018-03-28 00:37:34,655 : INFO : 121 batches submitted to accumulate stats from 7744 documents (627275 virtual) 2018-03-28 00:37:34,669 : INFO : 122 batches submitted to accumulate stats from 7808 documents (633329 virtual) 2018-03-28 00:37:34,681 : INFO : 123 batches submitted to accumulate stats from 7872 documents (638391 virtual) 2018-03-28 00:37:34,842 : INFO : 124 batches submitted to accumulate stats from 7936 documents (643487 virtual) 2018-03-28 00:37:34,868 : INFO : 125 batches submitted to accumulate stats from 8000 documents (647634 virtual) 2018-03-28 00:37:35,114 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:35,121 : INFO : accumulator serialized 2018-03-28 00:37:35,268 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:35,283 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:35,273 : INFO : accumulator serialized 2018-03-28 00:37:35,289 : INFO : accumulator serialized 2018-03-28 00:37:35,418 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:37:35,608 : INFO : accumulated word occurrence stats for 648305 virtual documents 2018-03-28 00:37:36,055 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_40 2018-03-28 00:37:36,057 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-28 00:37:36,061 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:37:36,061 : INFO : setting ignored attribute state to None 2018-03-28 00:37:36,062 : INFO : setting ignored attribute id2word to None 2018-03-28 00:37:36,062 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_40 2018-03-28 00:37:36,063 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_40.state 2018-03-28 00:37:36,082 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_40.state 2018-03-28 00:37:36,104 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:37:36,153 : INFO : 1 batches submitted to accumulate stats from 64 documents (4982 virtual) 2018-03-28 00:37:36,161 : INFO : 2 batches submitted to accumulate stats from 128 documents (10157 virtual) 2018-03-28 00:37:36,168 : INFO : 3 batches submitted to accumulate stats from 192 documents (15057 virtual) 2018-03-28 00:37:36,176 : INFO : 4 batches submitted to accumulate stats from 256 documents (19941 virtual) 2018-03-28 00:37:36,184 : INFO : 5 batches submitted to accumulate stats from 320 documents (25048 virtual) 2018-03-28 00:37:36,191 : INFO : 6 batches submitted to accumulate stats from 384 documents (30581 virtual) 2018-03-28 00:37:36,364 : INFO : 7 batches submitted to accumulate stats from 448 documents (35845 virtual) 2018-03-28 00:37:36,389 : INFO : 8 batches submitted to accumulate stats from 512 documents (40777 virtual) 2018-03-28 00:37:36,431 : INFO : 9 batches submitted to accumulate stats from 576 documents (45638 virtual) 2018-03-28 00:37:36,605 : INFO : 10 batches submitted to accumulate stats from 640 documents (51118 virtual) 2018-03-28 00:37:36,618 : INFO : 11 batches submitted to accumulate stats from 704 documents (56333 virtual) 2018-03-28 00:37:36,655 : INFO : 12 batches submitted to accumulate stats from 768 documents (61060 virtual) 2018-03-28 00:37:36,819 : INFO : 13 batches submitted to accumulate stats from 832 documents (66653 virtual) 2018-03-28 00:37:36,842 : INFO : 14 batches submitted to accumulate stats from 896 documents (72087 virtual) 2018-03-28 00:37:36,874 : INFO : 15 batches submitted to accumulate stats from 960 documents (77262 virtual) 2018-03-28 00:37:37,045 : INFO : 16 batches submitted to accumulate stats from 1024 documents (83294 virtual) 2018-03-28 00:37:37,069 : INFO : 17 batches submitted to accumulate stats from 1088 documents (88418 virtual) 2018-03-28 00:37:37,105 : INFO : 18 batches submitted to accumulate stats from 1152 documents (93238 virtual) 2018-03-28 00:37:37,302 : INFO : 19 batches submitted to accumulate stats from 1216 documents (98034 virtual) 2018-03-28 00:37:37,309 : INFO : 20 batches submitted to accumulate stats from 1280 documents (103767 virtual) 2018-03-28 00:37:37,316 : INFO : 21 batches submitted to accumulate stats from 1344 documents (109024 virtual) 2018-03-28 00:37:37,510 : INFO : 22 batches submitted to accumulate stats from 1408 documents (114387 virtual) 2018-03-28 00:37:37,540 : INFO : 23 batches submitted to accumulate stats from 1472 documents (120083 virtual) 2018-03-28 00:37:37,562 : INFO : 24 batches submitted to accumulate stats from 1536 documents (125131 virtual) 2018-03-28 00:37:37,732 : INFO : 25 batches submitted to accumulate stats from 1600 documents (130657 virtual) 2018-03-28 00:37:37,766 : INFO : 26 batches submitted to accumulate stats from 1664 documents (135536 virtual) 2018-03-28 00:37:37,793 : INFO : 27 batches submitted to accumulate stats from 1728 documents (140951 virtual) 2018-03-28 00:37:37,975 : INFO : 28 batches submitted to accumulate stats from 1792 documents (146121 virtual) 2018-03-28 00:37:37,996 : INFO : 29 batches submitted to accumulate stats from 1856 documents (151708 virtual) 2018-03-28 00:37:38,015 : INFO : 30 batches submitted to accumulate stats from 1920 documents (157298 virtual) 2018-03-28 00:37:38,208 : INFO : 31 batches submitted to accumulate stats from 1984 documents (162255 virtual) 2018-03-28 00:37:38,220 : INFO : 32 batches submitted to accumulate stats from 2048 documents (167206 virtual) 2018-03-28 00:37:38,245 : INFO : 33 batches submitted to accumulate stats from 2112 documents (172012 virtual) 2018-03-28 00:37:38,443 : INFO : 34 batches submitted to accumulate stats from 2176 documents (177432 virtual) 2018-03-28 00:37:38,462 : INFO : 35 batches submitted to accumulate stats from 2240 documents (182931 virtual) 2018-03-28 00:37:38,478 : INFO : 36 batches submitted to accumulate stats from 2304 documents (188013 virtual) 2018-03-28 00:37:38,655 : INFO : 37 batches submitted to accumulate stats from 2368 documents (193729 virtual) 2018-03-28 00:37:38,667 : INFO : 38 batches submitted to accumulate stats from 2432 documents (199970 virtual) 2018-03-28 00:37:38,713 : INFO : 39 batches submitted to accumulate stats from 2496 documents (205019 virtual) 2018-03-28 00:37:38,880 : INFO : 40 batches submitted to accumulate stats from 2560 documents (210023 virtual) 2018-03-28 00:37:38,896 : INFO : 41 batches submitted to accumulate stats from 2624 documents (214806 virtual) 2018-03-28 00:37:38,944 : INFO : 42 batches submitted to accumulate stats from 2688 documents (220000 virtual) 2018-03-28 00:37:39,121 : INFO : 43 batches submitted to accumulate stats from 2752 documents (224577 virtual) 2018-03-28 00:37:39,151 : INFO : 44 batches submitted to accumulate stats from 2816 documents (229091 virtual) 2018-03-28 00:37:39,163 : INFO : 45 batches submitted to accumulate stats from 2880 documents (234690 virtual) 2018-03-28 00:37:39,334 : INFO : 46 batches submitted to accumulate stats from 2944 documents (239521 virtual) 2018-03-28 00:37:39,367 : INFO : 47 batches submitted to accumulate stats from 3008 documents (244483 virtual) 2018-03-28 00:37:39,395 : INFO : 48 batches submitted to accumulate stats from 3072 documents (249446 virtual) 2018-03-28 00:37:39,521 : INFO : 49 batches submitted to accumulate stats from 3136 documents (254308 virtual) 2018-03-28 00:37:39,582 : INFO : 50 batches submitted to accumulate stats from 3200 documents (259741 virtual) 2018-03-28 00:37:39,620 : INFO : 51 batches submitted to accumulate stats from 3264 documents (264836 virtual) 2018-03-28 00:37:39,742 : INFO : 52 batches submitted to accumulate stats from 3328 documents (269983 virtual) 2018-03-28 00:37:39,827 : INFO : 53 batches submitted to accumulate stats from 3392 documents (275276 virtual) 2018-03-28 00:37:39,873 : INFO : 54 batches submitted to accumulate stats from 3456 documents (280281 virtual) 2018-03-28 00:37:39,998 : INFO : 55 batches submitted to accumulate stats from 3520 documents (285264 virtual) 2018-03-28 00:37:40,075 : INFO : 56 batches submitted to accumulate stats from 3584 documents (290232 virtual) 2018-03-28 00:37:40,093 : INFO : 57 batches submitted to accumulate stats from 3648 documents (295448 virtual) 2018-03-28 00:37:40,201 : INFO : 58 batches submitted to accumulate stats from 3712 documents (300819 virtual) 2018-03-28 00:37:40,308 : INFO : 59 batches submitted to accumulate stats from 3776 documents (305975 virtual) 2018-03-28 00:37:40,313 : INFO : 60 batches submitted to accumulate stats from 3840 documents (310984 virtual) 2018-03-28 00:37:40,413 : INFO : 61 batches submitted to accumulate stats from 3904 documents (316238 virtual) 2018-03-28 00:37:40,519 : INFO : 62 batches submitted to accumulate stats from 3968 documents (320871 virtual) 2018-03-28 00:37:40,541 : INFO : 63 batches submitted to accumulate stats from 4032 documents (325840 virtual) 2018-03-28 00:37:40,664 : INFO : 64 batches submitted to accumulate stats from 4096 documents (331190 virtual) 2018-03-28 00:37:40,751 : INFO : 65 batches submitted to accumulate stats from 4160 documents (336204 virtual) 2018-03-28 00:37:40,770 : INFO : 66 batches submitted to accumulate stats from 4224 documents (341653 virtual) 2018-03-28 00:37:40,890 : INFO : 67 batches submitted to accumulate stats from 4288 documents (346672 virtual) 2018-03-28 00:37:40,974 : INFO : 68 batches submitted to accumulate stats from 4352 documents (351560 virtual) 2018-03-28 00:37:40,985 : INFO : 69 batches submitted to accumulate stats from 4416 documents (356754 virtual) 2018-03-28 00:37:41,120 : INFO : 70 batches submitted to accumulate stats from 4480 documents (362007 virtual) 2018-03-28 00:37:41,196 : INFO : 71 batches submitted to accumulate stats from 4544 documents (366687 virtual) 2018-03-28 00:37:41,232 : INFO : 72 batches submitted to accumulate stats from 4608 documents (372025 virtual) 2018-03-28 00:37:41,307 : INFO : 73 batches submitted to accumulate stats from 4672 documents (377004 virtual) 2018-03-28 00:37:41,415 : INFO : 74 batches submitted to accumulate stats from 4736 documents (382675 virtual) 2018-03-28 00:37:41,488 : INFO : 75 batches submitted to accumulate stats from 4800 documents (387833 virtual) 2018-03-28 00:37:41,524 : INFO : 76 batches submitted to accumulate stats from 4864 documents (392595 virtual) 2018-03-28 00:37:41,620 : INFO : 77 batches submitted to accumulate stats from 4928 documents (397610 virtual) 2018-03-28 00:37:41,720 : INFO : 78 batches submitted to accumulate stats from 4992 documents (402897 virtual) 2018-03-28 00:37:41,733 : INFO : 79 batches submitted to accumulate stats from 5056 documents (407892 virtual) 2018-03-28 00:37:41,873 : INFO : 80 batches submitted to accumulate stats from 5120 documents (413289 virtual) 2018-03-28 00:37:41,927 : INFO : 81 batches submitted to accumulate stats from 5184 documents (418818 virtual) 2018-03-28 00:37:41,959 : INFO : 82 batches submitted to accumulate stats from 5248 documents (423833 virtual) 2018-03-28 00:37:42,081 : INFO : 83 batches submitted to accumulate stats from 5312 documents (429231 virtual) 2018-03-28 00:37:42,171 : INFO : 84 batches submitted to accumulate stats from 5376 documents (434923 virtual) 2018-03-28 00:37:42,192 : INFO : 85 batches submitted to accumulate stats from 5440 documents (440428 virtual) 2018-03-28 00:37:42,315 : INFO : 86 batches submitted to accumulate stats from 5504 documents (446295 virtual) 2018-03-28 00:37:42,444 : INFO : 87 batches submitted to accumulate stats from 5568 documents (450695 virtual) 2018-03-28 00:37:42,453 : INFO : 88 batches submitted to accumulate stats from 5632 documents (456089 virtual) 2018-03-28 00:37:42,557 : INFO : 89 batches submitted to accumulate stats from 5696 documents (461264 virtual) 2018-03-28 00:37:42,692 : INFO : 90 batches submitted to accumulate stats from 5760 documents (466660 virtual) 2018-03-28 00:37:42,699 : INFO : 91 batches submitted to accumulate stats from 5824 documents (472113 virtual) 2018-03-28 00:37:42,781 : INFO : 92 batches submitted to accumulate stats from 5888 documents (477564 virtual) 2018-03-28 00:37:42,902 : INFO : 93 batches submitted to accumulate stats from 5952 documents (482758 virtual) 2018-03-28 00:37:42,934 : INFO : 94 batches submitted to accumulate stats from 6016 documents (488334 virtual) 2018-03-28 00:37:42,993 : INFO : 95 batches submitted to accumulate stats from 6080 documents (493868 virtual) 2018-03-28 00:37:43,121 : INFO : 96 batches submitted to accumulate stats from 6144 documents (499242 virtual) 2018-03-28 00:37:43,154 : INFO : 97 batches submitted to accumulate stats from 6208 documents (504636 virtual) 2018-03-28 00:37:43,248 : INFO : 98 batches submitted to accumulate stats from 6272 documents (509567 virtual) 2018-03-28 00:37:43,331 : INFO : 99 batches submitted to accumulate stats from 6336 documents (514572 virtual) 2018-03-28 00:37:43,409 : INFO : 100 batches submitted to accumulate stats from 6400 documents (519486 virtual) 2018-03-28 00:37:43,487 : INFO : 101 batches submitted to accumulate stats from 6464 documents (524871 virtual) 2018-03-28 00:37:43,565 : INFO : 102 batches submitted to accumulate stats from 6528 documents (529588 virtual) 2018-03-28 00:37:43,669 : INFO : 103 batches submitted to accumulate stats from 6592 documents (534520 virtual) 2018-03-28 00:37:43,700 : INFO : 104 batches submitted to accumulate stats from 6656 documents (539677 virtual) 2018-03-28 00:37:43,785 : INFO : 105 batches submitted to accumulate stats from 6720 documents (545088 virtual) 2018-03-28 00:37:43,897 : INFO : 106 batches submitted to accumulate stats from 6784 documents (549949 virtual) 2018-03-28 00:37:43,904 : INFO : 107 batches submitted to accumulate stats from 6848 documents (555265 virtual) 2018-03-28 00:37:44,005 : INFO : 108 batches submitted to accumulate stats from 6912 documents (559967 virtual) 2018-03-28 00:37:44,119 : INFO : 109 batches submitted to accumulate stats from 6976 documents (565982 virtual) 2018-03-28 00:37:44,129 : INFO : 110 batches submitted to accumulate stats from 7040 documents (570710 virtual) 2018-03-28 00:37:44,230 : INFO : 111 batches submitted to accumulate stats from 7104 documents (575620 virtual) 2018-03-28 00:37:44,350 : INFO : 112 batches submitted to accumulate stats from 7168 documents (580261 virtual) 2018-03-28 00:37:44,357 : INFO : 113 batches submitted to accumulate stats from 7232 documents (585711 virtual) 2018-03-28 00:37:44,445 : INFO : 114 batches submitted to accumulate stats from 7296 documents (590386 virtual) 2018-03-28 00:37:44,572 : INFO : 115 batches submitted to accumulate stats from 7360 documents (595608 virtual) 2018-03-28 00:37:44,598 : INFO : 116 batches submitted to accumulate stats from 7424 documents (601103 virtual) 2018-03-28 00:37:44,655 : INFO : 117 batches submitted to accumulate stats from 7488 documents (606539 virtual) 2018-03-28 00:37:44,785 : INFO : 118 batches submitted to accumulate stats from 7552 documents (611440 virtual) 2018-03-28 00:37:44,849 : INFO : 119 batches submitted to accumulate stats from 7616 documents (616536 virtual) 2018-03-28 00:37:44,857 : INFO : 120 batches submitted to accumulate stats from 7680 documents (621619 virtual) 2018-03-28 00:37:45,005 : INFO : 121 batches submitted to accumulate stats from 7744 documents (626653 virtual) 2018-03-28 00:37:45,084 : INFO : 122 batches submitted to accumulate stats from 7808 documents (632365 virtual) 2018-03-28 00:37:45,124 : INFO : 123 batches submitted to accumulate stats from 7872 documents (637405 virtual) 2018-03-28 00:37:45,239 : INFO : 124 batches submitted to accumulate stats from 7936 documents (642818 virtual) 2018-03-28 00:37:45,267 : INFO : 125 batches submitted to accumulate stats from 8000 documents (647614 virtual) 2018-03-28 00:37:45,532 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:45,538 : INFO : accumulator serialized 2018-03-28 00:37:45,705 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:45,749 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:45,711 : INFO : accumulator serialized 2018-03-28 00:37:45,755 : INFO : accumulator serialized 2018-03-28 00:37:45,899 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:37:46,147 : INFO : accumulated word occurrence stats for 648317 virtual documents 2018-03-28 00:37:46,672 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_50 2018-03-28 00:37:46,675 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_50.expElogbeta.npy with mmap=None 2018-03-28 00:37:46,680 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:37:46,681 : INFO : setting ignored attribute state to None 2018-03-28 00:37:46,681 : INFO : setting ignored attribute id2word to None 2018-03-28 00:37:46,682 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_50 2018-03-28 00:37:46,682 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_50.state 2018-03-28 00:37:46,705 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_50.state 2018-03-28 00:37:46,732 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:37:46,781 : INFO : 1 batches submitted to accumulate stats from 64 documents (4982 virtual) 2018-03-28 00:37:46,790 : INFO : 2 batches submitted to accumulate stats from 128 documents (10157 virtual) 2018-03-28 00:37:46,795 : INFO : 3 batches submitted to accumulate stats from 192 documents (15057 virtual) 2018-03-28 00:37:46,803 : INFO : 4 batches submitted to accumulate stats from 256 documents (19941 virtual) 2018-03-28 00:37:46,811 : INFO : 5 batches submitted to accumulate stats from 320 documents (25048 virtual) 2018-03-28 00:37:46,819 : INFO : 6 batches submitted to accumulate stats from 384 documents (30581 virtual) 2018-03-28 00:37:47,010 : INFO : 7 batches submitted to accumulate stats from 448 documents (35845 virtual) 2018-03-28 00:37:47,039 : INFO : 8 batches submitted to accumulate stats from 512 documents (40777 virtual) 2018-03-28 00:37:47,067 : INFO : 9 batches submitted to accumulate stats from 576 documents (45638 virtual) 2018-03-28 00:37:47,264 : INFO : 10 batches submitted to accumulate stats from 640 documents (51118 virtual) 2018-03-28 00:37:47,271 : INFO : 11 batches submitted to accumulate stats from 704 documents (56333 virtual) 2018-03-28 00:37:47,323 : INFO : 12 batches submitted to accumulate stats from 768 documents (60983 virtual) 2018-03-28 00:37:47,490 : INFO : 13 batches submitted to accumulate stats from 832 documents (66643 virtual) 2018-03-28 00:37:47,535 : INFO : 14 batches submitted to accumulate stats from 896 documents (72016 virtual) 2018-03-28 00:37:47,550 : INFO : 15 batches submitted to accumulate stats from 960 documents (77174 virtual) 2018-03-28 00:37:47,697 : INFO : 16 batches submitted to accumulate stats from 1024 documents (83175 virtual) 2018-03-28 00:37:47,793 : INFO : 17 batches submitted to accumulate stats from 1088 documents (88314 virtual) 2018-03-28 00:37:47,804 : INFO : 18 batches submitted to accumulate stats from 1152 documents (93141 virtual) 2018-03-28 00:37:47,925 : INFO : 19 batches submitted to accumulate stats from 1216 documents (97904 virtual) 2018-03-28 00:37:48,036 : INFO : 20 batches submitted to accumulate stats from 1280 documents (103726 virtual) 2018-03-28 00:37:48,062 : INFO : 21 batches submitted to accumulate stats from 1344 documents (108954 virtual) 2018-03-28 00:37:48,198 : INFO : 22 batches submitted to accumulate stats from 1408 documents (114386 virtual) 2018-03-28 00:37:48,274 : INFO : 23 batches submitted to accumulate stats from 1472 documents (120015 virtual) 2018-03-28 00:37:48,285 : INFO : 24 batches submitted to accumulate stats from 1536 documents (125036 virtual) 2018-03-28 00:37:48,436 : INFO : 25 batches submitted to accumulate stats from 1600 documents (130590 virtual) 2018-03-28 00:37:48,513 : INFO : 26 batches submitted to accumulate stats from 1664 documents (135538 virtual) 2018-03-28 00:37:48,541 : INFO : 27 batches submitted to accumulate stats from 1728 documents (140904 virtual) 2018-03-28 00:37:48,667 : INFO : 28 batches submitted to accumulate stats from 1792 documents (146046 virtual) 2018-03-28 00:37:48,754 : INFO : 29 batches submitted to accumulate stats from 1856 documents (151365 virtual) 2018-03-28 00:37:48,806 : INFO : 30 batches submitted to accumulate stats from 1920 documents (157100 virtual) 2018-03-28 00:37:48,898 : INFO : 31 batches submitted to accumulate stats from 1984 documents (162022 virtual) 2018-03-28 00:37:48,986 : INFO : 32 batches submitted to accumulate stats from 2048 documents (167046 virtual) 2018-03-28 00:37:49,097 : INFO : 33 batches submitted to accumulate stats from 2112 documents (171918 virtual) 2018-03-28 00:37:49,119 : INFO : 34 batches submitted to accumulate stats from 2176 documents (177248 virtual) 2018-03-28 00:37:49,208 : INFO : 35 batches submitted to accumulate stats from 2240 documents (182757 virtual) 2018-03-28 00:37:49,336 : INFO : 36 batches submitted to accumulate stats from 2304 documents (187697 virtual) 2018-03-28 00:37:49,373 : INFO : 37 batches submitted to accumulate stats from 2368 documents (193401 virtual) 2018-03-28 00:37:49,469 : INFO : 38 batches submitted to accumulate stats from 2432 documents (199702 virtual) 2018-03-28 00:37:49,580 : INFO : 39 batches submitted to accumulate stats from 2496 documents (204584 virtual) 2018-03-28 00:37:49,600 : INFO : 40 batches submitted to accumulate stats from 2560 documents (209829 virtual) 2018-03-28 00:37:49,696 : INFO : 41 batches submitted to accumulate stats from 2624 documents (214652 virtual) 2018-03-28 00:37:49,819 : INFO : 42 batches submitted to accumulate stats from 2688 documents (219589 virtual) 2018-03-28 00:37:49,908 : INFO : 43 batches submitted to accumulate stats from 2752 documents (224425 virtual) 2018-03-28 00:37:50,042 : INFO : 44 batches submitted to accumulate stats from 2816 documents (228797 virtual) 2018-03-28 00:37:50,049 : INFO : 45 batches submitted to accumulate stats from 2880 documents (234337 virtual) 2018-03-28 00:37:50,142 : INFO : 46 batches submitted to accumulate stats from 2944 documents (239090 virtual) 2018-03-28 00:37:50,247 : INFO : 47 batches submitted to accumulate stats from 3008 documents (244163 virtual) 2018-03-28 00:37:50,272 : INFO : 48 batches submitted to accumulate stats from 3072 documents (249016 virtual) 2018-03-28 00:37:50,394 : INFO : 49 batches submitted to accumulate stats from 3136 documents (254021 virtual) 2018-03-28 00:37:50,431 : INFO : 50 batches submitted to accumulate stats from 3200 documents (259184 virtual) 2018-03-28 00:37:50,518 : INFO : 51 batches submitted to accumulate stats from 3264 documents (264615 virtual) 2018-03-28 00:37:50,629 : INFO : 52 batches submitted to accumulate stats from 3328 documents (269467 virtual) 2018-03-28 00:37:50,674 : INFO : 53 batches submitted to accumulate stats from 3392 documents (274797 virtual) 2018-03-28 00:37:50,777 : INFO : 54 batches submitted to accumulate stats from 3456 documents (279801 virtual) 2018-03-28 00:37:50,844 : INFO : 55 batches submitted to accumulate stats from 3520 documents (284585 virtual) 2018-03-28 00:37:50,928 : INFO : 56 batches submitted to accumulate stats from 3584 documents (289830 virtual) 2018-03-28 00:37:51,004 : INFO : 57 batches submitted to accumulate stats from 3648 documents (294996 virtual) 2018-03-28 00:37:51,067 : INFO : 58 batches submitted to accumulate stats from 3712 documents (300516 virtual) 2018-03-28 00:37:51,185 : INFO : 59 batches submitted to accumulate stats from 3776 documents (305642 virtual) 2018-03-28 00:37:51,234 : INFO : 60 batches submitted to accumulate stats from 3840 documents (310556 virtual) 2018-03-28 00:37:51,325 : INFO : 61 batches submitted to accumulate stats from 3904 documents (315832 virtual) 2018-03-28 00:37:51,413 : INFO : 62 batches submitted to accumulate stats from 3968 documents (320771 virtual) 2018-03-28 00:37:51,468 : INFO : 63 batches submitted to accumulate stats from 4032 documents (325491 virtual) 2018-03-28 00:37:51,559 : INFO : 64 batches submitted to accumulate stats from 4096 documents (330901 virtual) 2018-03-28 00:37:51,645 : INFO : 65 batches submitted to accumulate stats from 4160 documents (335837 virtual) 2018-03-28 00:37:51,735 : INFO : 66 batches submitted to accumulate stats from 4224 documents (341359 virtual) 2018-03-28 00:37:51,783 : INFO : 67 batches submitted to accumulate stats from 4288 documents (346349 virtual) 2018-03-28 00:37:51,877 : INFO : 68 batches submitted to accumulate stats from 4352 documents (351390 virtual) 2018-03-28 00:37:51,988 : INFO : 69 batches submitted to accumulate stats from 4416 documents (356421 virtual) 2018-03-28 00:37:52,035 : INFO : 70 batches submitted to accumulate stats from 4480 documents (361639 virtual) 2018-03-28 00:37:52,103 : INFO : 71 batches submitted to accumulate stats from 4544 documents (366641 virtual) 2018-03-28 00:37:52,227 : INFO : 72 batches submitted to accumulate stats from 4608 documents (371577 virtual) 2018-03-28 00:37:52,296 : INFO : 73 batches submitted to accumulate stats from 4672 documents (376647 virtual) 2018-03-28 00:37:52,384 : INFO : 74 batches submitted to accumulate stats from 4736 documents (382212 virtual) 2018-03-28 00:37:52,471 : INFO : 75 batches submitted to accumulate stats from 4800 documents (387502 virtual) 2018-03-28 00:37:52,570 : INFO : 76 batches submitted to accumulate stats from 4864 documents (392352 virtual) 2018-03-28 00:37:52,632 : INFO : 77 batches submitted to accumulate stats from 4928 documents (397254 virtual) 2018-03-28 00:37:52,678 : INFO : 78 batches submitted to accumulate stats from 4992 documents (402785 virtual) 2018-03-28 00:37:52,781 : INFO : 79 batches submitted to accumulate stats from 5056 documents (407587 virtual) 2018-03-28 00:37:52,913 : INFO : 80 batches submitted to accumulate stats from 5120 documents (412916 virtual) 2018-03-28 00:37:52,924 : INFO : 81 batches submitted to accumulate stats from 5184 documents (418346 virtual) 2018-03-28 00:37:53,015 : INFO : 82 batches submitted to accumulate stats from 5248 documents (423702 virtual) 2018-03-28 00:37:53,127 : INFO : 83 batches submitted to accumulate stats from 5312 documents (428973 virtual) 2018-03-28 00:37:53,180 : INFO : 84 batches submitted to accumulate stats from 5376 documents (434624 virtual) 2018-03-28 00:37:53,240 : INFO : 85 batches submitted to accumulate stats from 5440 documents (440230 virtual) 2018-03-28 00:37:53,398 : INFO : 86 batches submitted to accumulate stats from 5504 documents (446039 virtual) 2018-03-28 00:37:53,404 : INFO : 87 batches submitted to accumulate stats from 5568 documents (450373 virtual) 2018-03-28 00:37:53,520 : INFO : 88 batches submitted to accumulate stats from 5632 documents (455761 virtual) 2018-03-28 00:37:53,639 : INFO : 89 batches submitted to accumulate stats from 5696 documents (460897 virtual) 2018-03-28 00:37:53,658 : INFO : 90 batches submitted to accumulate stats from 5760 documents (466429 virtual) 2018-03-28 00:37:53,780 : INFO : 91 batches submitted to accumulate stats from 5824 documents (471884 virtual) 2018-03-28 00:37:53,871 : INFO : 92 batches submitted to accumulate stats from 5888 documents (477235 virtual) 2018-03-28 00:37:53,900 : INFO : 93 batches submitted to accumulate stats from 5952 documents (482371 virtual) 2018-03-28 00:37:54,027 : INFO : 94 batches submitted to accumulate stats from 6016 documents (487878 virtual) 2018-03-28 00:37:54,123 : INFO : 95 batches submitted to accumulate stats from 6080 documents (493691 virtual) 2018-03-28 00:37:54,136 : INFO : 96 batches submitted to accumulate stats from 6144 documents (498858 virtual) 2018-03-28 00:37:54,296 : INFO : 97 batches submitted to accumulate stats from 6208 documents (504486 virtual) 2018-03-28 00:37:54,369 : INFO : 98 batches submitted to accumulate stats from 6272 documents (509116 virtual) 2018-03-28 00:37:54,375 : INFO : 99 batches submitted to accumulate stats from 6336 documents (514222 virtual) 2018-03-28 00:37:54,549 : INFO : 100 batches submitted to accumulate stats from 6400 documents (519055 virtual) 2018-03-28 00:37:54,608 : INFO : 101 batches submitted to accumulate stats from 6464 documents (524415 virtual) 2018-03-28 00:37:54,630 : INFO : 102 batches submitted to accumulate stats from 6528 documents (529368 virtual) 2018-03-28 00:37:54,796 : INFO : 103 batches submitted to accumulate stats from 6592 documents (534395 virtual) 2018-03-28 00:37:54,850 : INFO : 104 batches submitted to accumulate stats from 6656 documents (539397 virtual) 2018-03-28 00:37:54,871 : INFO : 105 batches submitted to accumulate stats from 6720 documents (544799 virtual) 2018-03-28 00:37:55,000 : INFO : 106 batches submitted to accumulate stats from 6784 documents (549519 virtual) 2018-03-28 00:37:55,106 : INFO : 107 batches submitted to accumulate stats from 6848 documents (555051 virtual) 2018-03-28 00:37:55,121 : INFO : 108 batches submitted to accumulate stats from 6912 documents (559855 virtual) 2018-03-28 00:37:55,215 : INFO : 109 batches submitted to accumulate stats from 6976 documents (565802 virtual) 2018-03-28 00:37:55,343 : INFO : 110 batches submitted to accumulate stats from 7040 documents (570381 virtual) 2018-03-28 00:37:55,400 : INFO : 111 batches submitted to accumulate stats from 7104 documents (575391 virtual) 2018-03-28 00:37:55,428 : INFO : 112 batches submitted to accumulate stats from 7168 documents (580082 virtual) 2018-03-28 00:37:55,582 : INFO : 113 batches submitted to accumulate stats from 7232 documents (585352 virtual) 2018-03-28 00:37:55,637 : INFO : 114 batches submitted to accumulate stats from 7296 documents (590150 virtual) 2018-03-28 00:37:55,700 : INFO : 115 batches submitted to accumulate stats from 7360 documents (595359 virtual) 2018-03-28 00:37:55,798 : INFO : 116 batches submitted to accumulate stats from 7424 documents (600864 virtual) 2018-03-28 00:37:55,855 : INFO : 117 batches submitted to accumulate stats from 7488 documents (606299 virtual) 2018-03-28 00:37:55,925 : INFO : 118 batches submitted to accumulate stats from 7552 documents (611220 virtual) 2018-03-28 00:37:56,066 : INFO : 119 batches submitted to accumulate stats from 7616 documents (616369 virtual) 2018-03-28 00:37:56,081 : INFO : 120 batches submitted to accumulate stats from 7680 documents (621372 virtual) 2018-03-28 00:37:56,181 : INFO : 121 batches submitted to accumulate stats from 7744 documents (626419 virtual) 2018-03-28 00:37:56,330 : INFO : 122 batches submitted to accumulate stats from 7808 documents (632167 virtual) 2018-03-28 00:37:56,336 : INFO : 123 batches submitted to accumulate stats from 7872 documents (637255 virtual) 2018-03-28 00:37:56,413 : INFO : 124 batches submitted to accumulate stats from 7936 documents (642569 virtual) 2018-03-28 00:37:56,552 : INFO : 125 batches submitted to accumulate stats from 8000 documents (647585 virtual) 2018-03-28 00:37:56,586 : INFO : 126 batches submitted to accumulate stats from 8064 documents (647635 virtual) 2018-03-28 00:37:56,838 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:56,854 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:56,844 : INFO : accumulator serialized 2018-03-28 00:37:56,859 : INFO : accumulator serialized 2018-03-28 00:37:57,055 : INFO : serializing accumulator to return to master... 2018-03-28 00:37:57,061 : INFO : accumulator serialized 2018-03-28 00:37:57,238 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:37:57,560 : INFO : accumulated word occurrence stats for 648333 virtual documents
Best number of topics¶
In [51]:
tmfull_meta_score.set_index('num_topic')
tmfull_meta_score.to_csv( os.path.join('../src', 'tmfull_meta', 'scores.combined.csv'))
In [53]:
f = plt.figure(figsize=(8,8))
ax1 = f.add_subplot(211)
ax1 = tmfull_meta_score.plot(x='num_topic',y='c_v',ax=ax1, color='b', linestyle='-')
ax1.set_ylabel('c_v coherence score')
ax1 = tmfull_meta_score.plot(x='num_topic', y='u_mass', ax=ax1, secondary_y=True, color='r', linestyle='-.')
ax1.set_ylabel('u_mass coherence score')
ax2 = f.add_subplot(212)
ax2 = tmfull_meta_score.plot(x='num_topic', y='c_npmi', ax=ax2, color='g', linestyle='--')
ax2.set_ylabel('c_npmi coherence score')
ax2 = tmfull_meta_score.plot(x='num_topic', y='c_uci',ax=ax2, secondary_y=True, color='m',linestyle=':')
ax2.set_ylabel('u_uci coherence score')
# df = tmrest_meta_score
# ax = tmrest_meta_score.plot(x='num_topic')
# best = df.c_v.argmax()
# ax.scatter(x=df.num_topic.iloc[best], y=df.iloc[best].c_v, c='g', marker='*', s=150)
Out[53]:
Text(0,0.5,'u_uci coherence score')
In [54]:
f.savefig('./IMG/tmfull_meta.png', format='png', bbox_inches='tight')
Top terms for LDA¶
In [9]:
best_num_topics = 25
tmfull_meta_best_model = models.LdaModel.load(
'../src/tmfull_meta/ldamodel_nb_topics_{}'.format(best_num_topics))
2018-03-28 18:44:49,151 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_25 2018-03-28 18:44:49,449 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-28 18:44:49,476 : INFO : setting ignored attribute dispatcher to None 2018-03-28 18:44:49,477 : INFO : setting ignored attribute state to None 2018-03-28 18:44:49,477 : INFO : setting ignored attribute id2word to None 2018-03-28 18:44:49,478 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_25 2018-03-28 18:44:49,478 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_25.state 2018-03-28 18:44:49,523 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_25.state
In [6]:
tmfull_meta_best_model.save('../src/tmfull_meta/res/best_ldamodel')
2018-03-27 11:06:03,626 : INFO : saving LdaState object under ../src/tmfull_meta/res/best_ldamodel.state, separately None 2018-03-27 11:06:03,637 : INFO : saved ../src/tmfull_meta/res/best_ldamodel.state 2018-03-27 11:06:03,640 : INFO : saving LdaModel object under ../src/tmfull_meta/res/best_ldamodel, separately ['expElogbeta', 'sstats'] 2018-03-27 11:06:03,641 : INFO : storing np array 'expElogbeta' to ../src/tmfull_meta/res/best_ldamodel.expElogbeta.npy 2018-03-27 11:06:03,643 : INFO : not storing attribute dispatcher 2018-03-27 11:06:03,644 : INFO : not storing attribute state 2018-03-27 11:06:03,644 : INFO : not storing attribute id2word 2018-03-27 11:06:03,646 : INFO : saved ../src/tmfull_meta/res/best_ldamodel
In [82]:
for i in range(best_num_topics):
print('TOPIC {}'.format(i))
eg = tmfull_meta_best_model.get_topic_terms(i, topn=15)
for (a,b) in eg:
print('{}\t\t{:.5f}'.format(dict_tmfull_meta[a], b))
print('\n\n')
TOPIC 0 object 0.02948 image 0.02369 international 0.01600 publishing 0.01458 springer 0.01363 model 0.01133 conference 0.01068 class 0.01052 method 0.01040 vision 0.00905 computer 0.00882 segmentation 0.00788 computer_vision 0.00730 part 0.00721 recognition 0.00706 TOPIC 1 program 0.01166 instruction 0.01160 ieee 0.01113 performance 0.01035 compiler 0.00983 approach 0.00872 based 0.00864 algorithm 0.00863 technique 0.00819 time 0.00814 optimization 0.00778 processor 0.00767 using 0.00661 paper 0.00659 space 0.00635 TOPIC 2 speech 0.05230 model 0.01422 system 0.01400 synthesis 0.01370 speaker 0.01229 feature 0.01160 using 0.01089 acoustic 0.01068 based 0.01024 voice 0.00967 data 0.00807 recognition 0.00805 paper 0.00748 network 0.00728 speech_synthesis 0.00695 TOPIC 3 translation 0.05564 association 0.03400 machine 0.03009 computational 0.02951 linguistics 0.02679 system 0.02019 machine_translation 0.01781 association_computational 0.01723 proceeding 0.01509 language 0.01205 statistical 0.01181 task 0.01084 model 0.01026 association_computational_linguistics 0.00871 computational_linguistics 0.00870 TOPIC 4 system 0.02407 data 0.02008 acm 0.01986 user 0.01814 new 0.01316 application 0.01070 york 0.01043 new_york 0.01025 proceeding 0.00988 usa 0.00877 ny 0.00877 ny_usa 0.00869 access 0.00729 computing 0.00706 distributed 0.00667 TOPIC 5 research 0.01428 project 0.01101 technology 0.01064 web 0.01057 tool 0.00922 design 0.00912 paper 0.00884 system 0.00854 service 0.00830 computer 0.00755 science 0.00665 challenge 0.00649 data 0.00640 use 0.00612 support 0.00590 TOPIC 6 constraint 0.03917 logic 0.02341 dependency 0.02193 temporal 0.01509 key 0.01114 consistency 0.01112 order 0.01028 model 0.00941 problem 0.00879 first 0.00839 formula 0.00807 complete 0.00794 property 0.00793 show 0.00768 xml 0.00757 TOPIC 7 springer 0.07379 berlin 0.06248 heidelberg 0.05996 berlin_heidelberg 0.03474 springer_berlin 0.02520 international 0.02359 heidelberg_springer 0.01817 proceeding 0.01550 verlag 0.01411 conference 0.01244 springer_verlag 0.01181 paper 0.01179 gmbh 0.01070 model 0.00858 system 0.00800 TOPIC 8 language 0.02700 model 0.01624 corpus 0.01277 word 0.01274 text 0.01147 natural 0.01087 using 0.00909 dialogue 0.00845 natural_language 0.00837 method 0.00810 proceeding 0.00808 conference 0.00726 semantic 0.00722 computational 0.00696 grammar 0.00672 TOPIC 9 effect 0.01512 word 0.01447 cognitive 0.01340 model 0.01171 visual 0.00870 eye 0.00858 human 0.00821 processing 0.00788 study 0.00720 reading 0.00684 science 0.00640 two 0.00620 experiment 0.00609 information 0.00544 account 0.00524 TOPIC 10 quantum 0.03185 graph 0.03055 computation 0.01582 view 0.01264 pattern 0.01173 matching 0.00899 one 0.00862 algorithm 0.00853 measurement 0.00816 problem 0.00765 state 0.00726 classical 0.00699 based 0.00647 key 0.00643 protocol 0.00641 TOPIC 11 proof 0.02562 protocol 0.01195 security 0.01121 logic 0.01103 theorem 0.01084 formal 0.01012 system 0.00959 theory 0.00926 verification 0.00841 automated 0.00717 property 0.00676 reasoning 0.00660 method 0.00602 computer 0.00602 specification 0.00600 TOPIC 12 distribution 0.02052 algorithm 0.01579 scheme 0.01500 function 0.01469 signature 0.01021 bound 0.00866 result 0.00788 efficient 0.00786 time 0.00768 problem 0.00744 random 0.00742 number 0.00701 threshold 0.00656 optimal 0.00633 probability 0.00606 TOPIC 13 data 0.06704 query 0.05626 database 0.04121 provenance 0.01544 answering 0.01046 schema 0.00924 answer 0.00906 acm 0.00876 system 0.00802 information 0.00798 source 0.00727 relational 0.00703 proceeding 0.00660 rule 0.00646 query_answering 0.00611 TOPIC 14 model 0.01826 neural 0.01336 neuron 0.01266 network 0.01189 activity 0.00932 stimulus 0.00898 synaptic 0.00795 response 0.00698 brain 0.00681 cell 0.00629 neuroscience 0.00582 information 0.00577 population 0.00541 spike 0.00534 change 0.00525 TOPIC 15 user 0.02185 proceeding 0.01777 retrieval 0.01426 system 0.01267 social 0.01246 international 0.01245 information 0.01060 conference 0.00993 workshop 0.00956 test 0.00782 child 0.00747 topic 0.00720 content 0.00698 evaluation 0.00698 story 0.00675 TOPIC 16 learning 0.04307 model 0.03169 data 0.01312 method 0.01214 inference 0.01211 machine 0.01075 bayesian 0.01049 approach 0.00978 problem 0.00978 machine_learning 0.00953 probabilistic 0.00822 algorithm 0.00704 neural 0.00692 parameter 0.00665 network 0.00643 TOPIC 17 tree 0.02732 problem 0.01857 xml 0.01793 algorithm 0.01451 show 0.00937 automaton 0.00835 relation 0.00824 time 0.00810 class 0.00707 complexity 0.00691 given 0.00681 regular 0.00676 transducer 0.00641 grammar 0.00631 document 0.00630 TOPIC 18 agent 0.03228 system 0.02311 knowledge 0.01256 ontology 0.01190 interaction 0.01111 planning 0.00997 approach 0.00855 intelligence 0.00850 reasoning 0.00814 artificial 0.00807 based 0.00798 international 0.00772 paper 0.00762 domain 0.00759 proceeding 0.00745 TOPIC 19 image 0.01436 data 0.01319 search 0.01068 task 0.01016 visual 0.00799 based 0.00722 method 0.00707 model 0.00685 analysis 0.00675 classification 0.00672 video 0.00655 feature 0.00610 level 0.00595 using 0.00582 domain 0.00553 TOPIC 20 language 0.04077 programming 0.03583 type 0.03035 program 0.01911 semantics 0.01153 code 0.01124 functional 0.01057 system 0.00968 programming_language 0.00962 level 0.00878 parallel 0.00855 calculus 0.00811 implementation 0.00757 java 0.00624 acm 0.00621 TOPIC 21 control 0.02237 motion 0.02167 robot 0.02128 system 0.01199 method 0.01071 ieee 0.00918 dynamic 0.00916 based 0.00852 human 0.00809 body 0.00725 humanoid 0.00721 space 0.00693 using 0.00686 task 0.00612 movement 0.00609 TOPIC 22 network 0.02895 performance 0.01839 application 0.01124 memory 0.00951 energy 0.00782 ieee 0.00782 wireless 0.00769 architecture 0.00766 cache 0.00756 mobile 0.00730 high 0.00657 system 0.00651 using 0.00635 based 0.00539 show 0.00537 TOPIC 23 model 0.03220 system 0.03029 process 0.01852 stochastic 0.01392 algebra 0.01153 game 0.00983 modelling 0.00967 pepa 0.00859 state 0.00810 time 0.00765 computer 0.00744 rule 0.00724 analysis 0.00681 space 0.00665 science 0.00627 TOPIC 24 protein 0.01807 cell 0.01686 gene 0.01577 system 0.01049 expression 0.01037 biology 0.01015 model 0.00871 network 0.00855 molecular 0.00662 analysis 0.00635 using 0.00618 data 0.00618 pathway 0.00594 biological 0.00558 system_biology 0.00541
Get the $C_V$ score for each measure:¶
In [125]:
cv_score = tmfull_meta_best_model.top_topics(corpus_tmfull_meta, text_tmfull_meta, dict_tmfull_meta, coherence='c_v', topn=15)
2018-03-29 10:28:12,932 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-29 10:28:21,173 : INFO : serializing accumulator to return to master... 2018-03-29 10:28:21,177 : INFO : serializing accumulator to return to master... 2018-03-29 10:28:21,184 : INFO : accumulator serialized 2018-03-29 10:28:21,304 : INFO : serializing accumulator to return to master... 2018-03-29 10:28:21,179 : INFO : accumulator serialized 2018-03-29 10:28:21,311 : INFO : accumulator serialized 2018-03-29 10:28:21,432 : INFO : 3 accumulators retrieved from output queue 2018-03-29 10:28:21,600 : INFO : accumulated word occurrence stats for 91652 virtual documents
In [126]:
topics = [
3, 7, 14, 4, 24, 0, 2, 17, 20, 16, 11, 21, 13, 8, 18, 1, 5, 22, 19, 23,
9,12, 15, 6, 10
]
labels = [
'Machine Translation', 'IRR', 'Neuroscience', 'IRR', 'Bioinformatics',
'Computer Vision', 'Speech Synethesis', 'Relational Algebra (?)',
'Formal Language', 'Machine Learning', 'Formal Verification',
'Robot Control', 'Database System', 'NLP', 'Agent Based System',
'Compiler Optimisation', '??', 'Communication Network', '??',
'Computer simulation', 'Cognitive Science', 'Algorithm',
'Information Rerieval', 'Formal Logic (?)', 'Quantum Computing (?)'
]
assert len(topics) == len(labels), print("Topics: ", len(topics), "Labels: ", len(labels))
for i, (a, b) in enumerate(cv_score):
nb_topic = topics[i] +1 # because ldavis starts from 1 instead of 0
topic_label = labels[i]
_str = "\multirow{3}{*}{" + str(
nb_topic) + "} & \multirow{3}{*}{" + "{:.3f}".format(b) + "} "
start = True
for j, (prob, word) in enumerate(a):
_j = j + 1
_str += " & " + str(word.replace('_', '\_'))
if _j % 5 == 0:
if start:
start = False
_str += " & \multirow{3}{*}{" + str(topic_label) + "} \\\\ \n &"
elif _j == 15:
_str += " \\\\ \\midrule \n"
else:
_str += " \\\\ \n &"
print(_str)
\multirow{3}{*}{4} & \multirow{3}{*}{0.722} & translation & association & machine & computational & linguistics & \multirow{3}{*}{Machine Translation} \\
& & system & machine\_translation & association\_computational & proceeding & language \\
& & statistical & task & model & association\_computational\_linguistics & computational\_linguistics \\ \midrule
\multirow{3}{*}{8} & \multirow{3}{*}{0.689} & springer & berlin & heidelberg & berlin\_heidelberg & springer\_berlin & \multirow{3}{*}{IRR} \\
& & international & heidelberg\_springer & proceeding & verlag & conference \\
& & springer\_verlag & paper & gmbh & model & system \\ \midrule
\multirow{3}{*}{15} & \multirow{3}{*}{0.668} & model & neural & neuron & network & activity & \multirow{3}{*}{Neuroscience} \\
& & stimulus & synaptic & response & brain & cell \\
& & neuroscience & information & population & spike & change \\ \midrule
\multirow{3}{*}{5} & \multirow{3}{*}{0.622} & system & data & acm & user & new & \multirow{3}{*}{IRR} \\
& & application & york & new\_york & proceeding & usa \\
& & ny & ny\_usa & access & computing & distributed \\ \midrule
\multirow{3}{*}{25} & \multirow{3}{*}{0.611} & protein & cell & gene & system & expression & \multirow{3}{*}{Bioinformatics} \\
& & biology & model & network & molecular & analysis \\
& & using & data & pathway & biological & system\_biology \\ \midrule
\multirow{3}{*}{1} & \multirow{3}{*}{0.609} & object & image & international & publishing & springer & \multirow{3}{*}{Computer Vision} \\
& & model & conference & class & method & vision \\
& & computer & segmentation & computer\_vision & part & recognition \\ \midrule
\multirow{3}{*}{3} & \multirow{3}{*}{0.602} & speech & model & system & synthesis & speaker & \multirow{3}{*}{Speech Synethesis} \\
& & feature & using & acoustic & based & voice \\
& & data & recognition & paper & network & speech\_synthesis \\ \midrule
\multirow{3}{*}{18} & \multirow{3}{*}{0.595} & tree & problem & xml & algorithm & show & \multirow{3}{*}{Relational Algebra (?)} \\
& & automaton & relation & time & class & complexity \\
& & given & regular & transducer & grammar & document \\ \midrule
\multirow{3}{*}{21} & \multirow{3}{*}{0.576} & language & programming & type & program & semantics & \multirow{3}{*}{Formal Language} \\
& & code & functional & system & programming\_language & level \\
& & parallel & calculus & implementation & java & acm \\ \midrule
\multirow{3}{*}{17} & \multirow{3}{*}{0.566} & learning & model & data & method & inference & \multirow{3}{*}{Machine Learning} \\
& & machine & bayesian & approach & problem & machine\_learning \\
& & probabilistic & algorithm & neural & parameter & network \\ \midrule
\multirow{3}{*}{12} & \multirow{3}{*}{0.566} & proof & protocol & security & logic & theorem & \multirow{3}{*}{Formal Verification} \\
& & formal & system & theory & verification & automated \\
& & property & reasoning & method & computer & specification \\ \midrule
\multirow{3}{*}{22} & \multirow{3}{*}{0.556} & control & motion & robot & system & method & \multirow{3}{*}{Robot Control} \\
& & ieee & dynamic & based & human & body \\
& & humanoid & space & using & task & movement \\ \midrule
\multirow{3}{*}{14} & \multirow{3}{*}{0.553} & data & query & database & provenance & answering & \multirow{3}{*}{Database System} \\
& & schema & answer & acm & system & information \\
& & source & relational & proceeding & rule & query\_answering \\ \midrule
\multirow{3}{*}{9} & \multirow{3}{*}{0.551} & language & model & corpus & word & text & \multirow{3}{*}{NLP} \\
& & natural & using & dialogue & natural\_language & method \\
& & proceeding & conference & semantic & computational & grammar \\ \midrule
\multirow{3}{*}{19} & \multirow{3}{*}{0.528} & agent & system & knowledge & ontology & interaction & \multirow{3}{*}{Agent Based System} \\
& & planning & approach & intelligence & reasoning & artificial \\
& & based & international & paper & domain & proceeding \\ \midrule
\multirow{3}{*}{2} & \multirow{3}{*}{0.521} & program & instruction & ieee & performance & compiler & \multirow{3}{*}{Compiler Optimisation} \\
& & approach & based & algorithm & technique & time \\
& & optimization & processor & using & paper & space \\ \midrule
\multirow{3}{*}{6} & \multirow{3}{*}{0.504} & research & project & technology & web & tool & \multirow{3}{*}{??} \\
& & design & paper & system & service & computer \\
& & science & challenge & data & use & support \\ \midrule
\multirow{3}{*}{23} & \multirow{3}{*}{0.468} & network & performance & application & memory & energy & \multirow{3}{*}{Communication Network} \\
& & ieee & wireless & architecture & cache & mobile \\
& & high & system & using & based & show \\ \midrule
\multirow{3}{*}{20} & \multirow{3}{*}{0.447} & image & data & search & task & visual & \multirow{3}{*}{??} \\
& & based & method & model & analysis & classification \\
& & video & feature & level & using & domain \\ \midrule
\multirow{3}{*}{24} & \multirow{3}{*}{0.445} & model & system & process & stochastic & algebra & \multirow{3}{*}{Computer simulation} \\
& & game & modelling & pepa & state & time \\
& & computer & rule & analysis & space & science \\ \midrule
\multirow{3}{*}{10} & \multirow{3}{*}{0.444} & effect & word & cognitive & model & visual & \multirow{3}{*}{Cognitive Science} \\
& & eye & human & processing & study & reading \\
& & science & two & experiment & information & account \\ \midrule
\multirow{3}{*}{13} & \multirow{3}{*}{0.417} & distribution & algorithm & scheme & function & signature & \multirow{3}{*}{Algorithm} \\
& & bound & result & efficient & time & problem \\
& & random & number & threshold & optimal & probability \\ \midrule
\multirow{3}{*}{16} & \multirow{3}{*}{0.413} & user & proceeding & retrieval & system & social & \multirow{3}{*}{Information Rerieval} \\
& & international & information & conference & workshop & test \\
& & child & topic & content & evaluation & story \\ \midrule
\multirow{3}{*}{7} & \multirow{3}{*}{0.400} & constraint & logic & dependency & temporal & key & \multirow{3}{*}{Formal Logic (?)} \\
& & consistency & order & model & problem & first \\
& & formula & complete & property & show & xml \\ \midrule
\multirow{3}{*}{11} & \multirow{3}{*}{0.362} & quantum & graph & computation & view & pattern & \multirow{3}{*}{Quantum Computing (?)} \\
& & matching & one & algorithm & measurement & problem \\
& & state & classical & based & key & protocol \\ \midrule
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:27: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:27: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:27: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:27: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:27: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:27: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:27: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:27: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:27: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:27: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:27: DeprecationWarning: invalid escape sequence \m
<input>:18: DeprecationWarning: invalid escape sequence \m
<input>:19: DeprecationWarning: invalid escape sequence \m
<input>:23: DeprecationWarning: invalid escape sequence \_
<input>:27: DeprecationWarning: invalid escape sequence \m
<ipython-input-126-fe4588b549ac>:18: DeprecationWarning: invalid escape sequence \m
_str = "\multirow{3}{*}{" + str(
<ipython-input-126-fe4588b549ac>:19: DeprecationWarning: invalid escape sequence \m
nb_topic) + "} & \multirow{3}{*}{" + "{:.3f}".format(b) + "} "
<ipython-input-126-fe4588b549ac>:23: DeprecationWarning: invalid escape sequence \_
_str += " & " + str(word.replace('_', '\_'))
<ipython-input-126-fe4588b549ac>:27: DeprecationWarning: invalid escape sequence \m
_str += " & \multirow{3}{*}{" + str(topic_label) + "} \\\\ \n &"
In [67]:
probs = []
for (a,b) in cv_score:
probs.append(b)
mean_cv_score = np.mean(probs)
median_cv_score = np.median(probs)
sns.distplot(probs)
print(mean_cv_score, median_cv_score)
0.537411423836326 0.5534872390592016
/Users/weiting/miniconda3/envs/infnet3/lib/python3.6/site-packages/statsmodels/nonparametric/kde.py:475: DeprecationWarning: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer. grid,delta = np.linspace(a,b,gridsize,retstep=True)
Visualisation¶
In [127]:
tmfull_meta_best_model = models.LdaModel.load('../src/tmfull_meta/res/best_ldamodel')
2018-03-29 10:38:01,031 : INFO : loading LdaModel object from ../src/tmfull_meta/res/best_ldamodel 2018-03-29 10:38:01,045 : INFO : loading expElogbeta from ../src/tmfull_meta/res/best_ldamodel.expElogbeta.npy with mmap=None 2018-03-29 10:38:01,067 : INFO : setting ignored attribute dispatcher to None 2018-03-29 10:38:01,068 : INFO : setting ignored attribute state to None 2018-03-29 10:38:01,068 : INFO : setting ignored attribute id2word to None 2018-03-29 10:38:01,069 : INFO : loaded ../src/tmfull_meta/res/best_ldamodel 2018-03-29 10:38:01,069 : INFO : loading LdaModel object from ../src/tmfull_meta/res/best_ldamodel.state 2018-03-29 10:38:01,103 : INFO : loaded ../src/tmfull_meta/res/best_ldamodel.state
In [129]:
vis_data = gensimvis.prepare(
tmfull_meta_best_model,
corpus_tmfull_meta,
dict_tmfull_meta,
sort_topics=True,
mds='mmds')
/Users/weiting/miniconda3/envs/infnet3/lib/python3.6/site-packages/pyLDAvis/_prepare.py:387: DeprecationWarning: .ix is deprecated. Please use .loc for label based indexing or .iloc for positional indexing See the documentation here: http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated topic_term_dists = topic_term_dists.ix[topic_order]
In [77]:
pyLDAvis.display(vis_data)
Out[77]:
In [130]:
# save in html for viewing
pyLDAvis.save_html(vis_data, open('../src/tmfull_meta/res/ldavis.mmds.sort.html', 'w'))
2012-2017 Publications¶
tmrest : Metadata + PDF¶
In [10]:
dict_restricted = Dictionary.load(
os.path.join(DATA_DIR, 'corpora', 'dictionary.less.all'))
df_less_all = df_combined_toks.drop(
df_combined_toks[(df_combined_toks.year < 2012)
| (df_combined_toks.year > 2017)].index)
df_less_all['concat_toks'] = df_less_all.apply(
lambda row: row.toks_metada + row.toks_pdf2txt, axis=1)
# Create a bow tagging for each publication:
df_less_all['bow'] = df_less_all['concat_toks'].apply(dict_restricted.doc2bow)
# Generate a corpus based on the tokens, which we will be using later
corpus_tmrest = df_less_all.bow.tolist()
text_tmrest = df_less_all.concat_toks.tolist()
2018-03-26 16:21:53,011 : INFO : loading Dictionary object from ../../data/data_schoolofinf/corpora/dictionary.less.all 2018-03-26 16:21:53,035 : INFO : loaded ../../data/data_schoolofinf/corpora/dictionary.less.all
In [11]:
tmrest_full_score = pd.read_csv('../src/tmfull_meta/scores.csv')
In [12]:
tmrest_full_score
Out[12]:
| num_topic | c_v | u_mass | |
|---|---|---|---|
| 0 | 10 | 0.444709 | -2.032462 |
| 1 | 15 | 0.481778 | -1.970734 |
| 2 | 20 | 0.469636 | -2.193221 |
| 3 | 25 | 0.519118 | -2.108254 |
| 4 | 30 | 0.471171 | -2.511380 |
| 5 | 35 | 0.493119 | -2.439051 |
| 6 | 40 | 0.478231 | -2.966330 |
| 7 | 50 | 0.463822 | -3.264161 |
Evaluation of model¶
In [14]:
tmrest_full_score['c_v'] = tmrest_full_score.apply(lambda row: get_score(row.num_topic, dict_restricted, texts=text_tmrest, corpus=corpus_tmrest,model_dir='../src/tmrest/',
coherence='c_v'),
axis=1)
2018-03-26 16:40:11,770 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_10 2018-03-26 16:40:11,774 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-26 16:40:11,778 : INFO : setting ignored attribute dispatcher to None 2018-03-26 16:40:11,778 : INFO : setting ignored attribute id2word to None 2018-03-26 16:40:11,779 : INFO : setting ignored attribute state to None 2018-03-26 16:40:11,779 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_10 2018-03-26 16:40:11,780 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_10.state 2018-03-26 16:40:11,806 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_10.state 2018-03-26 16:40:11,827 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 16:40:11,942 : INFO : 1 batches submitted to accumulate stats from 64 documents (227778 virtual) 2018-03-26 16:40:12,043 : INFO : 2 batches submitted to accumulate stats from 128 documents (548738 virtual) 2018-03-26 16:40:12,140 : INFO : 3 batches submitted to accumulate stats from 192 documents (753719 virtual) 2018-03-26 16:40:12,262 : INFO : 4 batches submitted to accumulate stats from 256 documents (980318 virtual) 2018-03-26 16:40:12,378 : INFO : 5 batches submitted to accumulate stats from 320 documents (1180590 virtual) 2018-03-26 16:40:12,478 : INFO : 6 batches submitted to accumulate stats from 384 documents (1400404 virtual) 2018-03-26 16:40:18,512 : INFO : 7 batches submitted to accumulate stats from 448 documents (1615442 virtual) 2018-03-26 16:40:18,963 : INFO : 8 batches submitted to accumulate stats from 512 documents (1824992 virtual) 2018-03-26 16:40:21,681 : INFO : 9 batches submitted to accumulate stats from 576 documents (2091082 virtual) 2018-03-26 16:40:25,079 : INFO : 10 batches submitted to accumulate stats from 640 documents (2291798 virtual) 2018-03-26 16:40:25,437 : INFO : 11 batches submitted to accumulate stats from 704 documents (2463605 virtual) 2018-03-26 16:40:28,149 : INFO : 12 batches submitted to accumulate stats from 768 documents (2643956 virtual) 2018-03-26 16:40:31,275 : INFO : 13 batches submitted to accumulate stats from 832 documents (2888198 virtual) 2018-03-26 16:40:31,614 : INFO : 14 batches submitted to accumulate stats from 896 documents (3099109 virtual) 2018-03-26 16:40:35,811 : INFO : 15 batches submitted to accumulate stats from 960 documents (3372642 virtual) 2018-03-26 16:40:36,613 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3579740 virtual) 2018-03-26 16:40:37,041 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3821082 virtual) 2018-03-26 16:40:41,052 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4062901 virtual) 2018-03-26 16:40:43,322 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4273341 virtual) 2018-03-26 16:40:43,914 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4499886 virtual) 2018-03-26 16:40:49,079 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4708819 virtual) 2018-03-26 16:40:49,351 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4905829 virtual) 2018-03-26 16:40:50,897 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5104446 virtual) 2018-03-26 16:40:55,542 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5344612 virtual) 2018-03-26 16:40:56,198 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5556104 virtual) 2018-03-26 16:40:57,312 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5813766 virtual) 2018-03-26 16:41:01,554 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6031917 virtual) 2018-03-26 16:41:01,768 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6208719 virtual) 2018-03-26 16:41:03,117 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6443996 virtual) 2018-03-26 16:41:07,836 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6667897 virtual) 2018-03-26 16:41:08,503 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6879892 virtual) 2018-03-26 16:41:10,702 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7169915 virtual) 2018-03-26 16:41:13,699 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7365248 virtual) 2018-03-26 16:41:14,341 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7597080 virtual) 2018-03-26 16:41:17,503 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7767166 virtual) 2018-03-26 16:41:20,136 : INFO : 36 batches submitted to accumulate stats from 2304 documents (7984283 virtual) 2018-03-26 16:41:20,514 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8177138 virtual) 2018-03-26 16:41:25,793 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8416192 virtual) 2018-03-26 16:41:25,989 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8639285 virtual) 2018-03-26 16:41:27,294 : INFO : 40 batches submitted to accumulate stats from 2560 documents (8852513 virtual) 2018-03-26 16:41:30,809 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9035864 virtual) 2018-03-26 16:41:32,222 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9275827 virtual) 2018-03-26 16:41:33,022 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9585124 virtual) 2018-03-26 16:41:37,777 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9787886 virtual) 2018-03-26 16:41:38,855 : INFO : 45 batches submitted to accumulate stats from 2880 documents (9971283 virtual) 2018-03-26 16:41:39,087 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10324199 virtual) 2018-03-26 16:41:43,131 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10549707 virtual) 2018-03-26 16:41:45,974 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10761546 virtual) 2018-03-26 16:41:48,150 : INFO : 49 batches submitted to accumulate stats from 3136 documents (10946635 virtual) 2018-03-26 16:41:49,207 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11120306 virtual) 2018-03-26 16:41:51,107 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11536461 virtual) 2018-03-26 16:41:55,801 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11847990 virtual) 2018-03-26 16:41:57,224 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12097627 virtual) 2018-03-26 16:41:58,181 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12349111 virtual) 2018-03-26 16:42:01,312 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12555287 virtual) 2018-03-26 16:42:02,272 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12695709 virtual) 2018-03-26 16:42:14,135 : INFO : serializing accumulator to return to master... 2018-03-26 16:42:14,142 : INFO : accumulator serialized 2018-03-26 16:42:15,376 : INFO : serializing accumulator to return to master... 2018-03-26 16:42:15,382 : INFO : accumulator serialized 2018-03-26 16:42:16,037 : INFO : serializing accumulator to return to master... 2018-03-26 16:42:16,042 : INFO : accumulator serialized 2018-03-26 16:42:16,154 : INFO : 3 accumulators retrieved from output queue 2018-03-26 16:42:16,202 : INFO : accumulated word occurrence stats for 12725759 virtual documents 2018-03-26 16:42:16,565 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_15 2018-03-26 16:42:16,569 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_15.expElogbeta.npy with mmap=None 2018-03-26 16:42:16,575 : INFO : setting ignored attribute dispatcher to None 2018-03-26 16:42:16,576 : INFO : setting ignored attribute id2word to None 2018-03-26 16:42:16,577 : INFO : setting ignored attribute state to None 2018-03-26 16:42:16,577 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_15 2018-03-26 16:42:16,578 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_15.state 2018-03-26 16:42:16,615 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_15.state 2018-03-26 16:42:16,640 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 16:42:16,758 : INFO : 1 batches submitted to accumulate stats from 64 documents (224881 virtual) 2018-03-26 16:42:16,867 : INFO : 2 batches submitted to accumulate stats from 128 documents (546653 virtual) 2018-03-26 16:42:16,967 : INFO : 3 batches submitted to accumulate stats from 192 documents (749974 virtual) 2018-03-26 16:42:17,090 : INFO : 4 batches submitted to accumulate stats from 256 documents (964658 virtual) 2018-03-26 16:42:17,223 : INFO : 5 batches submitted to accumulate stats from 320 documents (1174756 virtual) 2018-03-26 16:42:17,337 : INFO : 6 batches submitted to accumulate stats from 384 documents (1382425 virtual) 2018-03-26 16:42:23,775 : INFO : 7 batches submitted to accumulate stats from 448 documents (1612138 virtual) 2018-03-26 16:42:24,551 : INFO : 8 batches submitted to accumulate stats from 512 documents (1798971 virtual) 2018-03-26 16:42:28,156 : INFO : 9 batches submitted to accumulate stats from 576 documents (2078950 virtual) 2018-03-26 16:42:31,106 : INFO : 10 batches submitted to accumulate stats from 640 documents (2275046 virtual) 2018-03-26 16:42:31,963 : INFO : 11 batches submitted to accumulate stats from 704 documents (2459958 virtual) 2018-03-26 16:42:35,169 : INFO : 12 batches submitted to accumulate stats from 768 documents (2634760 virtual) 2018-03-26 16:42:38,042 : INFO : 13 batches submitted to accumulate stats from 832 documents (2869966 virtual) 2018-03-26 16:42:38,855 : INFO : 14 batches submitted to accumulate stats from 896 documents (3077867 virtual) 2018-03-26 16:42:44,697 : INFO : 15 batches submitted to accumulate stats from 960 documents (3349606 virtual) 2018-03-26 16:42:44,857 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3564540 virtual) 2018-03-26 16:42:45,245 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3805284 virtual) 2018-03-26 16:42:50,374 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4041593 virtual) 2018-03-26 16:42:52,424 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4252802 virtual) 2018-03-26 16:42:53,246 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4470191 virtual) 2018-03-26 16:42:59,542 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4681789 virtual) 2018-03-26 16:42:59,663 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4866508 virtual) 2018-03-26 16:43:01,535 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5095645 virtual) 2018-03-26 16:43:07,037 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5319575 virtual) 2018-03-26 16:43:07,786 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5518487 virtual) 2018-03-26 16:43:08,986 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5742647 virtual) 2018-03-26 16:43:13,875 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5971768 virtual) 2018-03-26 16:43:14,190 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6169183 virtual) 2018-03-26 16:43:16,828 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6391319 virtual) 2018-03-26 16:43:21,200 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6607947 virtual) 2018-03-26 16:43:21,678 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6804998 virtual) 2018-03-26 16:43:24,634 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7024946 virtual) 2018-03-26 16:43:28,242 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7304411 virtual) 2018-03-26 16:43:29,298 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7516921 virtual) 2018-03-26 16:43:32,153 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7701119 virtual) 2018-03-26 16:43:35,852 : INFO : 36 batches submitted to accumulate stats from 2304 documents (7937384 virtual) 2018-03-26 16:43:36,133 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8112250 virtual) 2018-03-26 16:43:40,017 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8360482 virtual) 2018-03-26 16:43:43,418 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8569332 virtual) 2018-03-26 16:43:45,243 : INFO : 40 batches submitted to accumulate stats from 2560 documents (8796264 virtual) 2018-03-26 16:43:46,344 : INFO : 41 batches submitted to accumulate stats from 2624 documents (8967237 virtual) 2018-03-26 16:43:51,194 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9193029 virtual) 2018-03-26 16:43:51,641 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9515936 virtual) 2018-03-26 16:43:54,496 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9710329 virtual) 2018-03-26 16:43:58,708 : INFO : 45 batches submitted to accumulate stats from 2880 documents (9886425 virtual) 2018-03-26 16:43:59,254 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10197249 virtual) 2018-03-26 16:44:00,176 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10463739 virtual) 2018-03-26 16:44:06,471 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10704860 virtual) 2018-03-26 16:44:06,844 : INFO : 49 batches submitted to accumulate stats from 3136 documents (10875047 virtual) 2018-03-26 16:44:10,451 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11063859 virtual) 2018-03-26 16:44:12,642 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11409822 virtual) 2018-03-26 16:44:17,530 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11775146 virtual) 2018-03-26 16:44:19,113 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12019284 virtual) 2018-03-26 16:44:20,851 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12265090 virtual) 2018-03-26 16:44:23,042 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12447716 virtual) 2018-03-26 16:44:25,806 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12693958 virtual) 2018-03-26 16:44:40,128 : INFO : serializing accumulator to return to master... 2018-03-26 16:44:40,134 : INFO : accumulator serialized 2018-03-26 16:44:40,504 : INFO : serializing accumulator to return to master... 2018-03-26 16:44:40,511 : INFO : accumulator serialized 2018-03-26 16:44:42,301 : INFO : serializing accumulator to return to master... 2018-03-26 16:44:42,306 : INFO : accumulator serialized 2018-03-26 16:44:42,475 : INFO : 3 accumulators retrieved from output queue 2018-03-26 16:44:42,571 : INFO : accumulated word occurrence stats for 12725788 virtual documents 2018-03-26 16:44:43,106 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_20 2018-03-26 16:44:43,111 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_20.expElogbeta.npy with mmap=None 2018-03-26 16:44:43,117 : INFO : setting ignored attribute dispatcher to None 2018-03-26 16:44:43,118 : INFO : setting ignored attribute id2word to None 2018-03-26 16:44:43,118 : INFO : setting ignored attribute state to None 2018-03-26 16:44:43,119 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_20 2018-03-26 16:44:43,120 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_20.state 2018-03-26 16:44:43,165 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_20.state 2018-03-26 16:44:43,194 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 16:44:43,315 : INFO : 1 batches submitted to accumulate stats from 64 documents (223952 virtual) 2018-03-26 16:44:43,427 : INFO : 2 batches submitted to accumulate stats from 128 documents (546597 virtual) 2018-03-26 16:44:43,521 : INFO : 3 batches submitted to accumulate stats from 192 documents (744919 virtual) 2018-03-26 16:44:43,644 : INFO : 4 batches submitted to accumulate stats from 256 documents (958484 virtual) 2018-03-26 16:44:43,768 : INFO : 5 batches submitted to accumulate stats from 320 documents (1170061 virtual) 2018-03-26 16:44:43,889 : INFO : 6 batches submitted to accumulate stats from 384 documents (1371189 virtual) 2018-03-26 16:44:51,609 : INFO : 7 batches submitted to accumulate stats from 448 documents (1599055 virtual) 2018-03-26 16:44:52,278 : INFO : 8 batches submitted to accumulate stats from 512 documents (1790256 virtual) 2018-03-26 16:44:56,460 : INFO : 9 batches submitted to accumulate stats from 576 documents (2068965 virtual) 2018-03-26 16:44:59,743 : INFO : 10 batches submitted to accumulate stats from 640 documents (2269631 virtual) 2018-03-26 16:45:00,538 : INFO : 11 batches submitted to accumulate stats from 704 documents (2459802 virtual) 2018-03-26 16:45:03,996 : INFO : 12 batches submitted to accumulate stats from 768 documents (2632134 virtual) 2018-03-26 16:45:07,499 : INFO : 13 batches submitted to accumulate stats from 832 documents (2865895 virtual) 2018-03-26 16:45:07,921 : INFO : 14 batches submitted to accumulate stats from 896 documents (3076582 virtual) 2018-03-26 16:45:14,515 : INFO : 15 batches submitted to accumulate stats from 960 documents (3341861 virtual) 2018-03-26 16:45:14,944 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3552777 virtual) 2018-03-26 16:45:15,151 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3799094 virtual) 2018-03-26 16:45:20,663 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4037511 virtual) 2018-03-26 16:45:22,923 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4241242 virtual) 2018-03-26 16:45:23,794 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4466681 virtual) 2018-03-26 16:45:30,272 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4676087 virtual) 2018-03-26 16:45:30,545 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4861394 virtual) 2018-03-26 16:45:32,931 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5088375 virtual) 2018-03-26 16:45:38,263 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5305948 virtual) 2018-03-26 16:45:39,270 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5506841 virtual) 2018-03-26 16:45:41,435 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5729443 virtual) 2018-03-26 16:45:46,185 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5949167 virtual) 2018-03-26 16:45:46,635 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6163948 virtual) 2018-03-26 16:45:49,834 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6384236 virtual) 2018-03-26 16:45:54,187 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6597309 virtual) 2018-03-26 16:45:54,723 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6796411 virtual) 2018-03-26 16:45:57,932 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7024633 virtual) 2018-03-26 16:46:02,468 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7289123 virtual) 2018-03-26 16:46:02,591 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7503742 virtual) 2018-03-26 16:46:06,374 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7695683 virtual) 2018-03-26 16:46:10,256 : INFO : 36 batches submitted to accumulate stats from 2304 documents (7928378 virtual) 2018-03-26 16:46:10,865 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8108597 virtual) 2018-03-26 16:46:15,198 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8343013 virtual) 2018-03-26 16:46:19,361 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8558225 virtual) 2018-03-26 16:46:21,196 : INFO : 40 batches submitted to accumulate stats from 2560 documents (8784425 virtual) 2018-03-26 16:46:22,170 : INFO : 41 batches submitted to accumulate stats from 2624 documents (8956060 virtual) 2018-03-26 16:46:28,805 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9179925 virtual) 2018-03-26 16:46:29,086 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9500292 virtual) 2018-03-26 16:46:32,108 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9692846 virtual) 2018-03-26 16:46:38,261 : INFO : 45 batches submitted to accumulate stats from 2880 documents (9881686 virtual) 2018-03-26 16:46:39,028 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10175576 virtual) 2018-03-26 16:46:39,201 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10439342 virtual) 2018-03-26 16:46:46,853 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10694444 virtual) 2018-03-26 16:46:47,733 : INFO : 49 batches submitted to accumulate stats from 3136 documents (10850424 virtual) 2018-03-26 16:46:51,692 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11039038 virtual) 2018-03-26 16:46:54,440 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11311533 virtual) 2018-03-26 16:47:00,059 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11763913 virtual) 2018-03-26 16:47:02,158 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11993261 virtual) 2018-03-26 16:47:04,714 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12236944 virtual) 2018-03-26 16:47:05,894 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12422562 virtual) 2018-03-26 16:47:09,457 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12669101 virtual) 2018-03-26 16:47:13,890 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12693201 virtual) 2018-03-26 16:47:24,168 : INFO : serializing accumulator to return to master... 2018-03-26 16:47:24,176 : INFO : accumulator serialized 2018-03-26 16:47:25,003 : INFO : serializing accumulator to return to master... 2018-03-26 16:47:25,009 : INFO : accumulator serialized 2018-03-26 16:47:29,439 : INFO : serializing accumulator to return to master... 2018-03-26 16:47:29,444 : INFO : accumulator serialized 2018-03-26 16:47:29,682 : INFO : 3 accumulators retrieved from output queue 2018-03-26 16:47:29,842 : INFO : accumulated word occurrence stats for 12725796 virtual documents 2018-03-26 16:47:30,566 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_25 2018-03-26 16:47:30,635 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-26 16:47:30,642 : INFO : setting ignored attribute dispatcher to None 2018-03-26 16:47:30,642 : INFO : setting ignored attribute id2word to None 2018-03-26 16:47:30,643 : INFO : setting ignored attribute state to None 2018-03-26 16:47:30,643 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_25 2018-03-26 16:47:30,644 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_25.state 2018-03-26 16:47:30,705 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_25.state 2018-03-26 16:47:30,760 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 16:47:30,900 : INFO : 1 batches submitted to accumulate stats from 64 documents (223952 virtual) 2018-03-26 16:47:31,014 : INFO : 2 batches submitted to accumulate stats from 128 documents (546597 virtual) 2018-03-26 16:47:31,114 : INFO : 3 batches submitted to accumulate stats from 192 documents (744919 virtual) 2018-03-26 16:47:31,242 : INFO : 4 batches submitted to accumulate stats from 256 documents (958484 virtual) 2018-03-26 16:47:31,370 : INFO : 5 batches submitted to accumulate stats from 320 documents (1170061 virtual) 2018-03-26 16:47:31,489 : INFO : 6 batches submitted to accumulate stats from 384 documents (1371189 virtual) 2018-03-26 16:47:39,677 : INFO : 7 batches submitted to accumulate stats from 448 documents (1599055 virtual) 2018-03-26 16:47:40,894 : INFO : 8 batches submitted to accumulate stats from 512 documents (1790256 virtual) 2018-03-26 16:47:45,992 : INFO : 9 batches submitted to accumulate stats from 576 documents (2068965 virtual) 2018-03-26 16:47:49,259 : INFO : 10 batches submitted to accumulate stats from 640 documents (2269631 virtual) 2018-03-26 16:47:50,899 : INFO : 11 batches submitted to accumulate stats from 704 documents (2459802 virtual) 2018-03-26 16:47:55,272 : INFO : 12 batches submitted to accumulate stats from 768 documents (2632134 virtual) 2018-03-26 16:47:59,106 : INFO : 13 batches submitted to accumulate stats from 832 documents (2865895 virtual) 2018-03-26 16:47:59,313 : INFO : 14 batches submitted to accumulate stats from 896 documents (3072170 virtual) 2018-03-26 16:48:08,077 : INFO : 15 batches submitted to accumulate stats from 960 documents (3337595 virtual) 2018-03-26 16:48:08,273 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3550766 virtual) 2018-03-26 16:48:08,803 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3779839 virtual) 2018-03-26 16:48:15,745 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4030010 virtual) 2018-03-26 16:48:18,318 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4230924 virtual) 2018-03-26 16:48:19,577 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4453199 virtual) 2018-03-26 16:48:26,906 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4665416 virtual) 2018-03-26 16:48:27,764 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4854820 virtual) 2018-03-26 16:48:29,416 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5074776 virtual) 2018-03-26 16:48:36,387 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5294928 virtual) 2018-03-26 16:48:37,517 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5482426 virtual) 2018-03-26 16:48:38,589 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5719123 virtual) 2018-03-26 16:48:45,235 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5942660 virtual) 2018-03-26 16:48:46,116 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6158498 virtual) 2018-03-26 16:48:47,786 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6363863 virtual) 2018-03-26 16:48:55,035 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6567897 virtual) 2018-03-26 16:48:55,171 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6777598 virtual) 2018-03-26 16:48:58,620 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7008505 virtual) 2018-03-26 16:49:05,138 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7268601 virtual) 2018-03-26 16:49:06,061 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7495694 virtual) 2018-03-26 16:49:08,540 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7681043 virtual) 2018-03-26 16:49:15,765 : INFO : 36 batches submitted to accumulate stats from 2304 documents (7905809 virtual) 2018-03-26 16:49:16,365 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8085135 virtual) 2018-03-26 16:49:20,214 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8335420 virtual) 2018-03-26 16:49:27,798 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8545967 virtual) 2018-03-26 16:49:29,106 : INFO : 40 batches submitted to accumulate stats from 2560 documents (8768677 virtual) 2018-03-26 16:49:29,272 : INFO : 41 batches submitted to accumulate stats from 2624 documents (8946581 virtual) 2018-03-26 16:49:38,422 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9159091 virtual) 2018-03-26 16:49:39,203 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9490959 virtual) 2018-03-26 16:49:41,480 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9671200 virtual) 2018-03-26 16:49:48,778 : INFO : 45 batches submitted to accumulate stats from 2880 documents (9870712 virtual) 2018-03-26 16:49:49,691 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10133173 virtual) 2018-03-26 16:49:50,662 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10422268 virtual) 2018-03-26 16:49:59,240 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10638249 virtual) 2018-03-26 16:49:59,575 : INFO : 49 batches submitted to accumulate stats from 3136 documents (10830638 virtual) 2018-03-26 16:50:05,950 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11028032 virtual) 2018-03-26 16:50:08,627 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11259844 virtual) 2018-03-26 16:50:12,541 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11610843 virtual) 2018-03-26 16:50:19,572 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11954456 virtual) 2018-03-26 16:50:20,066 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12197877 virtual) 2018-03-26 16:50:21,886 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12380815 virtual) 2018-03-26 16:50:29,949 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12640901 virtual) 2018-03-26 16:50:30,364 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12692425 virtual) 2018-03-26 16:50:48,199 : INFO : serializing accumulator to return to master... 2018-03-26 16:50:48,225 : INFO : accumulator serialized 2018-03-26 16:50:48,699 : INFO : serializing accumulator to return to master... 2018-03-26 16:50:48,705 : INFO : accumulator serialized 2018-03-26 16:50:52,043 : INFO : serializing accumulator to return to master... 2018-03-26 16:50:52,049 : INFO : accumulator serialized 2018-03-26 16:50:52,388 : INFO : 3 accumulators retrieved from output queue 2018-03-26 16:50:52,620 : INFO : accumulated word occurrence stats for 12725804 virtual documents 2018-03-26 16:50:53,644 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_30 2018-03-26 16:50:53,685 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-26 16:50:53,693 : INFO : setting ignored attribute dispatcher to None 2018-03-26 16:50:53,694 : INFO : setting ignored attribute id2word to None 2018-03-26 16:50:53,695 : INFO : setting ignored attribute state to None 2018-03-26 16:50:53,695 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_30 2018-03-26 16:50:53,696 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_30.state 2018-03-26 16:50:53,765 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_30.state 2018-03-26 16:50:53,805 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 16:50:53,927 : INFO : 1 batches submitted to accumulate stats from 64 documents (223952 virtual) 2018-03-26 16:50:54,041 : INFO : 2 batches submitted to accumulate stats from 128 documents (546597 virtual) 2018-03-26 16:50:54,145 : INFO : 3 batches submitted to accumulate stats from 192 documents (744919 virtual) 2018-03-26 16:50:54,277 : INFO : 4 batches submitted to accumulate stats from 256 documents (958484 virtual) 2018-03-26 16:50:54,407 : INFO : 5 batches submitted to accumulate stats from 320 documents (1170061 virtual) 2018-03-26 16:50:54,528 : INFO : 6 batches submitted to accumulate stats from 384 documents (1366595 virtual) 2018-03-26 16:51:05,231 : INFO : 7 batches submitted to accumulate stats from 448 documents (1592023 virtual) 2018-03-26 16:51:07,065 : INFO : 8 batches submitted to accumulate stats from 512 documents (1790227 virtual) 2018-03-26 16:51:12,656 : INFO : 9 batches submitted to accumulate stats from 576 documents (2068056 virtual) 2018-03-26 16:51:17,030 : INFO : 10 batches submitted to accumulate stats from 640 documents (2268404 virtual) 2018-03-26 16:51:19,603 : INFO : 11 batches submitted to accumulate stats from 704 documents (2459181 virtual) 2018-03-26 16:51:23,866 : INFO : 12 batches submitted to accumulate stats from 768 documents (2626954 virtual) 2018-03-26 16:51:29,053 : INFO : 13 batches submitted to accumulate stats from 832 documents (2861201 virtual) 2018-03-26 16:51:30,138 : INFO : 14 batches submitted to accumulate stats from 896 documents (3072180 virtual) 2018-03-26 16:51:38,853 : INFO : 15 batches submitted to accumulate stats from 960 documents (3337605 virtual) 2018-03-26 16:51:39,713 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3552684 virtual) 2018-03-26 16:51:40,378 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3779846 virtual) 2018-03-26 16:51:47,492 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4030017 virtual) 2018-03-26 16:51:51,982 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4230931 virtual) 2018-03-26 16:51:53,023 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4453206 virtual) 2018-03-26 16:52:01,548 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4665423 virtual) 2018-03-26 16:52:03,434 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4854827 virtual) 2018-03-26 16:52:04,434 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5074783 virtual) 2018-03-26 16:52:13,123 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5294935 virtual) 2018-03-26 16:52:13,234 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5482433 virtual) 2018-03-26 16:52:14,313 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5719130 virtual) 2018-03-26 16:52:21,847 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5942667 virtual) 2018-03-26 16:52:22,036 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6158505 virtual) 2018-03-26 16:52:24,056 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6359319 virtual) 2018-03-26 16:52:30,864 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6567893 virtual) 2018-03-26 16:52:31,481 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6777594 virtual) 2018-03-26 16:52:34,670 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7004706 virtual) 2018-03-26 16:52:40,839 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7264353 virtual) 2018-03-26 16:52:41,445 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7491620 virtual) 2018-03-26 16:52:43,823 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7679239 virtual) 2018-03-26 16:52:50,624 : INFO : 36 batches submitted to accumulate stats from 2304 documents (7905717 virtual) 2018-03-26 16:52:51,011 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8085038 virtual) 2018-03-26 16:52:54,270 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8324238 virtual) 2018-03-26 16:53:01,703 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8540676 virtual) 2018-03-26 16:53:02,668 : INFO : 40 batches submitted to accumulate stats from 2560 documents (8753423 virtual) 2018-03-26 16:53:02,821 : INFO : 41 batches submitted to accumulate stats from 2624 documents (8937975 virtual) 2018-03-26 16:53:10,973 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9159088 virtual) 2018-03-26 16:53:12,251 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9490956 virtual) 2018-03-26 16:53:12,950 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9671197 virtual) 2018-03-26 16:53:20,859 : INFO : 45 batches submitted to accumulate stats from 2880 documents (9870709 virtual) 2018-03-26 16:53:21,014 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10133170 virtual) 2018-03-26 16:53:22,048 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10422265 virtual) 2018-03-26 16:53:29,930 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10642452 virtual) 2018-03-26 16:53:30,753 : INFO : 49 batches submitted to accumulate stats from 3136 documents (10835661 virtual) 2018-03-26 16:53:35,776 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11029591 virtual) 2018-03-26 16:53:39,393 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11269169 virtual) 2018-03-26 16:53:43,193 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11610863 virtual) 2018-03-26 16:53:48,617 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11961059 virtual) 2018-03-26 16:53:49,801 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12206836 virtual) 2018-03-26 16:53:51,099 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12385406 virtual) 2018-03-26 16:53:57,424 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12640967 virtual) 2018-03-26 16:53:59,091 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12692517 virtual) 2018-03-26 16:54:13,711 : INFO : serializing accumulator to return to master... 2018-03-26 16:54:13,730 : INFO : accumulator serialized 2018-03-26 16:54:14,774 : INFO : serializing accumulator to return to master... 2018-03-26 16:54:14,780 : INFO : accumulator serialized 2018-03-26 16:54:18,470 : INFO : serializing accumulator to return to master... 2018-03-26 16:54:18,476 : INFO : accumulator serialized 2018-03-26 16:54:18,819 : INFO : 3 accumulators retrieved from output queue 2018-03-26 16:54:19,109 : INFO : accumulated word occurrence stats for 12725803 virtual documents 2018-03-26 16:54:20,186 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_35 2018-03-26 16:54:20,221 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_35.expElogbeta.npy with mmap=None 2018-03-26 16:54:20,230 : INFO : setting ignored attribute dispatcher to None 2018-03-26 16:54:20,231 : INFO : setting ignored attribute id2word to None 2018-03-26 16:54:20,232 : INFO : setting ignored attribute state to None 2018-03-26 16:54:20,232 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_35 2018-03-26 16:54:20,233 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_35.state 2018-03-26 16:54:20,455 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_35.state 2018-03-26 16:54:20,495 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 16:54:20,617 : INFO : 1 batches submitted to accumulate stats from 64 documents (223952 virtual) 2018-03-26 16:54:20,736 : INFO : 2 batches submitted to accumulate stats from 128 documents (546597 virtual) 2018-03-26 16:54:20,831 : INFO : 3 batches submitted to accumulate stats from 192 documents (749982 virtual) 2018-03-26 16:54:20,966 : INFO : 4 batches submitted to accumulate stats from 256 documents (958488 virtual) 2018-03-26 16:54:21,114 : INFO : 5 batches submitted to accumulate stats from 320 documents (1174717 virtual) 2018-03-26 16:54:21,238 : INFO : 6 batches submitted to accumulate stats from 384 documents (1371203 virtual) 2018-03-26 16:54:30,726 : INFO : 7 batches submitted to accumulate stats from 448 documents (1599069 virtual) 2018-03-26 16:54:31,749 : INFO : 8 batches submitted to accumulate stats from 512 documents (1790270 virtual) 2018-03-26 16:54:37,484 : INFO : 9 batches submitted to accumulate stats from 576 documents (2068979 virtual) 2018-03-26 16:54:41,178 : INFO : 10 batches submitted to accumulate stats from 640 documents (2269645 virtual) 2018-03-26 16:54:42,623 : INFO : 11 batches submitted to accumulate stats from 704 documents (2459816 virtual) 2018-03-26 16:54:47,497 : INFO : 12 batches submitted to accumulate stats from 768 documents (2632148 virtual) 2018-03-26 16:54:51,799 : INFO : 13 batches submitted to accumulate stats from 832 documents (2865909 virtual) 2018-03-26 16:54:51,925 : INFO : 14 batches submitted to accumulate stats from 896 documents (3072184 virtual) 2018-03-26 16:55:00,951 : INFO : 15 batches submitted to accumulate stats from 960 documents (3337609 virtual) 2018-03-26 16:55:01,319 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3550780 virtual) 2018-03-26 16:55:01,501 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3779755 virtual) 2018-03-26 16:55:09,212 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4025300 virtual) 2018-03-26 16:55:11,547 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4226504 virtual) 2018-03-26 16:55:13,530 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4453142 virtual) 2018-03-26 16:55:21,806 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4660563 virtual) 2018-03-26 16:55:22,301 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4854835 virtual) 2018-03-26 16:55:25,292 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5067646 virtual) 2018-03-26 16:55:33,111 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5291951 virtual) 2018-03-26 16:55:34,811 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5475855 virtual) 2018-03-26 16:55:36,476 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5718177 virtual) 2018-03-26 16:55:42,386 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5938550 virtual) 2018-03-26 16:55:43,974 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6158464 virtual) 2018-03-26 16:55:46,442 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6356197 virtual) 2018-03-26 16:55:53,110 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6560412 virtual) 2018-03-26 16:55:53,470 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6770971 virtual) 2018-03-26 16:55:58,073 : INFO : 32 batches submitted to accumulate stats from 2048 documents (6994312 virtual) 2018-03-26 16:56:03,702 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7255896 virtual) 2018-03-26 16:56:04,249 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7486250 virtual) 2018-03-26 16:56:07,527 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7660462 virtual) 2018-03-26 16:56:13,945 : INFO : 36 batches submitted to accumulate stats from 2304 documents (7899774 virtual) 2018-03-26 16:56:14,043 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8066686 virtual) 2018-03-26 16:56:18,399 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8316018 virtual) 2018-03-26 16:56:25,311 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8529182 virtual) 2018-03-26 16:56:27,011 : INFO : 40 batches submitted to accumulate stats from 2560 documents (8742046 virtual) 2018-03-26 16:56:27,267 : INFO : 41 batches submitted to accumulate stats from 2624 documents (8921313 virtual) 2018-03-26 16:56:35,266 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9147843 virtual) 2018-03-26 16:56:36,845 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9484581 virtual) 2018-03-26 16:56:39,206 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9664066 virtual) 2018-03-26 16:56:45,821 : INFO : 45 batches submitted to accumulate stats from 2880 documents (9867851 virtual) 2018-03-26 16:56:47,101 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10113612 virtual) 2018-03-26 16:56:47,739 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10410989 virtual) 2018-03-26 16:56:56,590 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10617173 virtual) 2018-03-26 16:56:56,862 : INFO : 49 batches submitted to accumulate stats from 3136 documents (10815454 virtual) 2018-03-26 16:57:03,914 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11019944 virtual) 2018-03-26 16:57:06,343 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11246981 virtual) 2018-03-26 16:57:08,381 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11595646 virtual) 2018-03-26 16:57:16,200 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11953358 virtual) 2018-03-26 16:57:17,219 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12179722 virtual) 2018-03-26 16:57:17,864 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12375513 virtual) 2018-03-26 16:57:26,451 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12615977 virtual) 2018-03-26 16:57:26,643 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12692041 virtual) 2018-03-26 16:57:43,898 : INFO : serializing accumulator to return to master... 2018-03-26 16:57:44,046 : INFO : accumulator serialized 2018-03-26 16:57:45,704 : INFO : serializing accumulator to return to master... 2018-03-26 16:57:45,710 : INFO : accumulator serialized 2018-03-26 16:57:47,329 : INFO : serializing accumulator to return to master... 2018-03-26 16:57:47,336 : INFO : accumulator serialized 2018-03-26 16:57:47,754 : INFO : 3 accumulators retrieved from output queue 2018-03-26 16:57:48,104 : INFO : accumulated word occurrence stats for 12725808 virtual documents 2018-03-26 16:57:49,367 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_40 2018-03-26 16:57:49,411 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-26 16:57:49,533 : INFO : setting ignored attribute dispatcher to None 2018-03-26 16:57:49,534 : INFO : setting ignored attribute id2word to None 2018-03-26 16:57:49,534 : INFO : setting ignored attribute state to None 2018-03-26 16:57:49,535 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_40 2018-03-26 16:57:49,535 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_40.state 2018-03-26 16:57:49,777 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_40.state 2018-03-26 16:57:49,839 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 16:57:49,961 : INFO : 1 batches submitted to accumulate stats from 64 documents (223952 virtual) 2018-03-26 16:57:50,076 : INFO : 2 batches submitted to accumulate stats from 128 documents (546597 virtual) 2018-03-26 16:57:50,181 : INFO : 3 batches submitted to accumulate stats from 192 documents (749982 virtual) 2018-03-26 16:57:50,307 : INFO : 4 batches submitted to accumulate stats from 256 documents (958488 virtual) 2018-03-26 16:57:50,438 : INFO : 5 batches submitted to accumulate stats from 320 documents (1174717 virtual) 2018-03-26 16:57:50,557 : INFO : 6 batches submitted to accumulate stats from 384 documents (1371203 virtual) 2018-03-26 16:58:00,290 : INFO : 7 batches submitted to accumulate stats from 448 documents (1599069 virtual) 2018-03-26 16:58:01,751 : INFO : 8 batches submitted to accumulate stats from 512 documents (1790246 virtual) 2018-03-26 16:58:07,987 : INFO : 9 batches submitted to accumulate stats from 576 documents (2068075 virtual) 2018-03-26 16:58:11,551 : INFO : 10 batches submitted to accumulate stats from 640 documents (2268423 virtual) 2018-03-26 16:58:13,395 : INFO : 11 batches submitted to accumulate stats from 704 documents (2459200 virtual) 2018-03-26 16:58:17,803 : INFO : 12 batches submitted to accumulate stats from 768 documents (2626973 virtual) 2018-03-26 16:58:22,190 : INFO : 13 batches submitted to accumulate stats from 832 documents (2861220 virtual) 2018-03-26 16:58:22,405 : INFO : 14 batches submitted to accumulate stats from 896 documents (3069039 virtual) 2018-03-26 16:58:31,623 : INFO : 15 batches submitted to accumulate stats from 960 documents (3337578 virtual) 2018-03-26 16:58:31,756 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3550665 virtual) 2018-03-26 16:58:31,900 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3779772 virtual) 2018-03-26 16:58:39,561 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4025317 virtual) 2018-03-26 16:58:42,226 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4226521 virtual) 2018-03-26 16:58:43,916 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4453229 virtual) 2018-03-26 16:58:52,881 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4665446 virtual) 2018-03-26 16:58:52,999 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4856190 virtual) 2018-03-26 16:58:55,368 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5074932 virtual) 2018-03-26 16:59:03,227 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5297493 virtual) 2018-03-26 16:59:05,204 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5493400 virtual) 2018-03-26 16:59:06,671 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5719304 virtual) 2018-03-26 16:59:12,791 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5942778 virtual) 2018-03-26 16:59:14,563 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6158596 virtual) 2018-03-26 16:59:17,285 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6363891 virtual) 2018-03-26 16:59:23,520 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6574759 virtual) 2018-03-26 16:59:24,407 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6780955 virtual) 2018-03-26 16:59:28,378 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7008525 virtual) 2018-03-26 16:59:34,895 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7268626 virtual) 2018-03-26 16:59:35,029 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7495719 virtual) 2018-03-26 16:59:38,525 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7681068 virtual) 2018-03-26 16:59:44,705 : INFO : 36 batches submitted to accumulate stats from 2304 documents (7905834 virtual) 2018-03-26 16:59:45,749 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8085160 virtual) 2018-03-26 16:59:49,839 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8335445 virtual) 2018-03-26 16:59:56,914 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8545992 virtual) 2018-03-26 16:59:57,828 : INFO : 40 batches submitted to accumulate stats from 2560 documents (8768702 virtual) 2018-03-26 16:59:59,321 : INFO : 41 batches submitted to accumulate stats from 2624 documents (8946606 virtual) 2018-03-26 17:00:07,157 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9159116 virtual) 2018-03-26 17:00:08,392 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9490984 virtual) 2018-03-26 17:00:11,279 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9671225 virtual) 2018-03-26 17:00:17,657 : INFO : 45 batches submitted to accumulate stats from 2880 documents (9870737 virtual) 2018-03-26 17:00:19,400 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10133198 virtual) 2018-03-26 17:00:19,836 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10422293 virtual) 2018-03-26 17:00:28,181 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10638274 virtual) 2018-03-26 17:00:28,791 : INFO : 49 batches submitted to accumulate stats from 3136 documents (10830663 virtual) 2018-03-26 17:00:35,907 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11028001 virtual) 2018-03-26 17:00:37,760 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11259762 virtual) 2018-03-26 17:00:41,741 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11603845 virtual) 2018-03-26 17:00:49,200 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11954377 virtual) 2018-03-26 17:00:49,856 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12194552 virtual) 2018-03-26 17:00:51,437 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12380684 virtual) 2018-03-26 17:00:59,256 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12636501 virtual) 2018-03-26 17:00:59,649 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12692346 virtual) 2018-03-26 17:01:17,338 : INFO : serializing accumulator to return to master... 2018-03-26 17:01:17,395 : INFO : accumulator serialized 2018-03-26 17:01:18,416 : INFO : serializing accumulator to return to master... 2018-03-26 17:01:18,422 : INFO : accumulator serialized 2018-03-26 17:01:21,535 : INFO : serializing accumulator to return to master... 2018-03-26 17:01:21,541 : INFO : accumulator serialized 2018-03-26 17:01:22,050 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:01:22,491 : INFO : accumulated word occurrence stats for 12725805 virtual documents 2018-03-26 17:01:24,009 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_50 2018-03-26 17:01:24,085 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_50.expElogbeta.npy with mmap=None 2018-03-26 17:01:24,265 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:01:24,265 : INFO : setting ignored attribute id2word to None 2018-03-26 17:01:24,266 : INFO : setting ignored attribute state to None 2018-03-26 17:01:24,266 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_50 2018-03-26 17:01:24,267 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_50.state 2018-03-26 17:01:24,625 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_50.state 2018-03-26 17:01:24,709 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:01:24,833 : INFO : 1 batches submitted to accumulate stats from 64 documents (223952 virtual) 2018-03-26 17:01:24,951 : INFO : 2 batches submitted to accumulate stats from 128 documents (546597 virtual) 2018-03-26 17:01:25,052 : INFO : 3 batches submitted to accumulate stats from 192 documents (749982 virtual) 2018-03-26 17:01:25,188 : INFO : 4 batches submitted to accumulate stats from 256 documents (958488 virtual) 2018-03-26 17:01:25,332 : INFO : 5 batches submitted to accumulate stats from 320 documents (1174717 virtual) 2018-03-26 17:01:25,450 : INFO : 6 batches submitted to accumulate stats from 384 documents (1371203 virtual) 2018-03-26 17:01:36,928 : INFO : 7 batches submitted to accumulate stats from 448 documents (1599069 virtual) 2018-03-26 17:01:38,506 : INFO : 8 batches submitted to accumulate stats from 512 documents (1790246 virtual) 2018-03-26 17:01:45,036 : INFO : 9 batches submitted to accumulate stats from 576 documents (2068075 virtual) 2018-03-26 17:01:49,958 : INFO : 10 batches submitted to accumulate stats from 640 documents (2259488 virtual) 2018-03-26 17:01:52,360 : INFO : 11 batches submitted to accumulate stats from 704 documents (2459099 virtual) 2018-03-26 17:01:57,068 : INFO : 12 batches submitted to accumulate stats from 768 documents (2626028 virtual) 2018-03-26 17:02:03,493 : INFO : 13 batches submitted to accumulate stats from 832 documents (2861086 virtual) 2018-03-26 17:02:03,708 : INFO : 14 batches submitted to accumulate stats from 896 documents (3066362 virtual) 2018-03-26 17:02:14,235 : INFO : 15 batches submitted to accumulate stats from 960 documents (3330003 virtual) 2018-03-26 17:02:15,510 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3550570 virtual) 2018-03-26 17:02:15,714 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3779635 virtual) 2018-03-26 17:02:24,155 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4015964 virtual) 2018-03-26 17:02:28,196 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4224615 virtual) 2018-03-26 17:02:30,362 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4441973 virtual) 2018-03-26 17:02:39,617 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4654655 virtual) 2018-03-26 17:02:41,379 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4846168 virtual) 2018-03-26 17:02:44,220 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5054531 virtual) 2018-03-26 17:02:54,155 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5277879 virtual) 2018-03-26 17:02:54,407 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5460766 virtual) 2018-03-26 17:02:57,237 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5711639 virtual) 2018-03-26 17:03:05,273 : INFO : 27 batches submitted to accumulate stats from 1728 documents (5938335 virtual) 2018-03-26 17:03:05,797 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6145251 virtual) 2018-03-26 17:03:08,996 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6351990 virtual) 2018-03-26 17:03:16,982 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6531312 virtual) 2018-03-26 17:03:18,214 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6760021 virtual) 2018-03-26 17:03:23,889 : INFO : 32 batches submitted to accumulate stats from 2048 documents (6974200 virtual) 2018-03-26 17:03:30,714 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7245122 virtual) 2018-03-26 17:03:32,002 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7462157 virtual) 2018-03-26 17:03:36,292 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7649432 virtual) 2018-03-26 17:03:41,690 : INFO : 36 batches submitted to accumulate stats from 2304 documents (7882609 virtual) 2018-03-26 17:03:45,162 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8038786 virtual) 2018-03-26 17:03:48,853 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8293039 virtual) 2018-03-26 17:03:57,847 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8498397 virtual) 2018-03-26 17:03:58,410 : INFO : 40 batches submitted to accumulate stats from 2560 documents (8713582 virtual) 2018-03-26 17:04:00,182 : INFO : 41 batches submitted to accumulate stats from 2624 documents (8905039 virtual) 2018-03-26 17:04:07,858 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9118369 virtual) 2018-03-26 17:04:11,578 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9449287 virtual) 2018-03-26 17:04:14,608 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9638029 virtual) 2018-03-26 17:04:20,419 : INFO : 45 batches submitted to accumulate stats from 2880 documents (9848908 virtual) 2018-03-26 17:04:24,867 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10081705 virtual) 2018-03-26 17:04:25,691 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10391693 virtual) 2018-03-26 17:04:32,799 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10607022 virtual) 2018-03-26 17:04:36,778 : INFO : 49 batches submitted to accumulate stats from 3136 documents (10795871 virtual) 2018-03-26 17:04:45,037 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11008048 virtual) 2018-03-26 17:04:45,162 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11238878 virtual) 2018-03-26 17:04:51,023 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11574316 virtual) 2018-03-26 17:04:57,773 : INFO : 53 batches submitted to accumulate stats from 3392 documents (11916479 virtual) 2018-03-26 17:05:01,697 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12147789 virtual) 2018-03-26 17:05:01,844 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12368845 virtual) 2018-03-26 17:05:10,797 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12582464 virtual) 2018-03-26 17:05:13,807 : INFO : 57 batches submitted to accumulate stats from 3648 documents (12691397 virtual) 2018-03-26 17:05:34,170 : INFO : serializing accumulator to return to master... 2018-03-26 17:05:34,213 : INFO : accumulator serialized 2018-03-26 17:05:37,014 : INFO : serializing accumulator to return to master... 2018-03-26 17:05:38,548 : INFO : serializing accumulator to return to master... 2018-03-26 17:05:38,556 : INFO : accumulator serialized 2018-03-26 17:05:37,021 : INFO : accumulator serialized 2018-03-26 17:05:40,181 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:05:40,813 : INFO : accumulated word occurrence stats for 12725815 virtual documents
In [15]:
tmrest_full_score['u_mass'] = tmrest_full_score.apply(lambda row: get_score(row.num_topic, dict_restricted, texts=text_tmrest, corpus=corpus_tmrest,model_dir='../src/tmrest/',
coherence='u_mass'),
axis=1)
2018-03-26 17:10:15,361 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_10 2018-03-26 17:10:15,452 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-26 17:10:15,493 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:10:15,494 : INFO : setting ignored attribute id2word to None 2018-03-26 17:10:15,494 : INFO : setting ignored attribute state to None 2018-03-26 17:10:15,495 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_10 2018-03-26 17:10:15,495 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_10.state 2018-03-26 17:10:15,570 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_10.state 2018-03-26 17:10:15,755 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-26 17:10:15,917 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-26 17:10:16,088 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-26 17:10:16,260 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_15 2018-03-26 17:10:16,293 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_15.expElogbeta.npy with mmap=None 2018-03-26 17:10:16,375 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:10:16,375 : INFO : setting ignored attribute id2word to None 2018-03-26 17:10:16,376 : INFO : setting ignored attribute state to None 2018-03-26 17:10:16,376 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_15 2018-03-26 17:10:16,377 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_15.state 2018-03-26 17:10:16,516 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_15.state 2018-03-26 17:10:16,754 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-26 17:10:16,921 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-26 17:10:17,101 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-26 17:10:17,321 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_20 2018-03-26 17:10:17,340 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_20.expElogbeta.npy with mmap=None 2018-03-26 17:10:17,417 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:10:17,418 : INFO : setting ignored attribute id2word to None 2018-03-26 17:10:17,418 : INFO : setting ignored attribute state to None 2018-03-26 17:10:17,418 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_20 2018-03-26 17:10:17,419 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_20.state 2018-03-26 17:10:17,547 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_20.state 2018-03-26 17:10:17,755 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-26 17:10:17,923 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-26 17:10:18,096 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-26 17:10:18,329 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_25 2018-03-26 17:10:18,397 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-26 17:10:18,548 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:10:18,549 : INFO : setting ignored attribute id2word to None 2018-03-26 17:10:18,549 : INFO : setting ignored attribute state to None 2018-03-26 17:10:18,550 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_25 2018-03-26 17:10:18,550 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_25.state 2018-03-26 17:10:18,745 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_25.state 2018-03-26 17:10:18,955 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-26 17:10:19,121 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-26 17:10:19,307 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-26 17:10:19,574 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_30 2018-03-26 17:10:19,623 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-26 17:10:19,717 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:10:19,718 : INFO : setting ignored attribute id2word to None 2018-03-26 17:10:19,718 : INFO : setting ignored attribute state to None 2018-03-26 17:10:19,719 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_30 2018-03-26 17:10:19,719 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_30.state 2018-03-26 17:10:19,948 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_30.state 2018-03-26 17:10:20,159 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-26 17:10:20,330 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-26 17:10:20,511 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-26 17:10:20,795 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_35 2018-03-26 17:10:20,826 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_35.expElogbeta.npy with mmap=None 2018-03-26 17:10:20,929 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:10:20,930 : INFO : setting ignored attribute id2word to None 2018-03-26 17:10:20,930 : INFO : setting ignored attribute state to None 2018-03-26 17:10:20,931 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_35 2018-03-26 17:10:20,932 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_35.state 2018-03-26 17:10:21,148 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_35.state 2018-03-26 17:10:21,371 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-26 17:10:21,552 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-26 17:10:21,753 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-26 17:10:22,053 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_40 2018-03-26 17:10:22,077 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-26 17:10:22,200 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:10:22,201 : INFO : setting ignored attribute id2word to None 2018-03-26 17:10:22,201 : INFO : setting ignored attribute state to None 2018-03-26 17:10:22,202 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_40 2018-03-26 17:10:22,202 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_40.state 2018-03-26 17:10:22,445 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_40.state 2018-03-26 17:10:22,704 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-26 17:10:22,891 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-26 17:10:23,084 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-26 17:10:23,415 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_50 2018-03-26 17:10:23,458 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_50.expElogbeta.npy with mmap=None 2018-03-26 17:10:23,605 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:10:23,606 : INFO : setting ignored attribute id2word to None 2018-03-26 17:10:23,606 : INFO : setting ignored attribute state to None 2018-03-26 17:10:23,606 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_50 2018-03-26 17:10:23,607 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_50.state 2018-03-26 17:10:23,978 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_50.state 2018-03-26 17:10:24,233 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-26 17:10:24,419 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-26 17:10:24,621 : INFO : CorpusAccumulator accumulated stats from 3000 documents
In [16]:
tmrest_full_score['c_npmi'] = tmrest_full_score.apply(lambda row: get_score(row.num_topic, dict_restricted, texts=text_tmrest, corpus=corpus_tmrest,model_dir='../src/tmrest/',
coherence='c_npmi'),
axis=1)
2018-03-26 17:10:24,998 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_10 2018-03-26 17:10:25,002 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-26 17:10:25,006 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:10:25,007 : INFO : setting ignored attribute id2word to None 2018-03-26 17:10:25,008 : INFO : setting ignored attribute state to None 2018-03-26 17:10:25,008 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_10 2018-03-26 17:10:25,009 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_10.state 2018-03-26 17:10:25,035 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_10.state 2018-03-26 17:10:25,057 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:10:25,174 : INFO : 1 batches submitted to accumulate stats from 64 documents (234178 virtual) 2018-03-26 17:10:25,285 : INFO : 2 batches submitted to accumulate stats from 128 documents (561538 virtual) 2018-03-26 17:10:25,379 : INFO : 3 batches submitted to accumulate stats from 192 documents (772919 virtual) 2018-03-26 17:10:25,507 : INFO : 4 batches submitted to accumulate stats from 256 documents (1005918 virtual) 2018-03-26 17:10:25,618 : INFO : 5 batches submitted to accumulate stats from 320 documents (1212590 virtual) 2018-03-26 17:10:25,740 : INFO : 6 batches submitted to accumulate stats from 384 documents (1438804 virtual) 2018-03-26 17:10:29,680 : INFO : 7 batches submitted to accumulate stats from 448 documents (1660242 virtual) 2018-03-26 17:10:29,881 : INFO : 8 batches submitted to accumulate stats from 512 documents (1876192 virtual) 2018-03-26 17:10:31,780 : INFO : 9 batches submitted to accumulate stats from 576 documents (2148682 virtual) 2018-03-26 17:10:33,699 : INFO : 10 batches submitted to accumulate stats from 640 documents (2355798 virtual) 2018-03-26 17:10:34,099 : INFO : 11 batches submitted to accumulate stats from 704 documents (2534005 virtual) 2018-03-26 17:10:35,547 : INFO : 12 batches submitted to accumulate stats from 768 documents (2720756 virtual) 2018-03-26 17:10:37,546 : INFO : 13 batches submitted to accumulate stats from 832 documents (2971398 virtual) 2018-03-26 17:10:37,808 : INFO : 14 batches submitted to accumulate stats from 896 documents (3188709 virtual) 2018-03-26 17:10:40,288 : INFO : 15 batches submitted to accumulate stats from 960 documents (3468642 virtual) 2018-03-26 17:10:40,803 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3682140 virtual) 2018-03-26 17:10:41,260 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3929882 virtual) 2018-03-26 17:10:43,473 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4178101 virtual) 2018-03-26 17:10:45,204 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4394941 virtual) 2018-03-26 17:10:45,374 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4627886 virtual) 2018-03-26 17:10:48,555 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4843219 virtual) 2018-03-26 17:10:48,905 : INFO : 22 batches submitted to accumulate stats from 1408 documents (5046629 virtual) 2018-03-26 17:10:49,717 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5251646 virtual) 2018-03-26 17:10:52,777 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5498212 virtual) 2018-03-26 17:10:52,958 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5716104 virtual) 2018-03-26 17:10:53,753 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5980166 virtual) 2018-03-26 17:10:56,359 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6204717 virtual) 2018-03-26 17:10:56,775 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6387919 virtual) 2018-03-26 17:10:57,426 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6629596 virtual) 2018-03-26 17:11:00,614 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6859897 virtual) 2018-03-26 17:11:00,734 : INFO : 31 batches submitted to accumulate stats from 1984 documents (7078292 virtual) 2018-03-26 17:11:02,121 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7374715 virtual) 2018-03-26 17:11:03,830 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7576448 virtual) 2018-03-26 17:11:04,754 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7814680 virtual) 2018-03-26 17:11:06,356 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7991166 virtual) 2018-03-26 17:11:08,026 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8214683 virtual) 2018-03-26 17:11:08,138 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8413938 virtual) 2018-03-26 17:11:11,334 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8659392 virtual) 2018-03-26 17:11:11,486 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8888885 virtual) 2018-03-26 17:11:12,359 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9108513 virtual) 2018-03-26 17:11:14,507 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9298264 virtual) 2018-03-26 17:11:15,372 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9544627 virtual) 2018-03-26 17:11:15,918 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9860324 virtual) 2018-03-26 17:11:18,856 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10069486 virtual) 2018-03-26 17:11:19,324 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10259283 virtual) 2018-03-26 17:11:19,654 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10618599 virtual) 2018-03-26 17:11:22,538 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10850507 virtual) 2018-03-26 17:11:24,561 : INFO : 48 batches submitted to accumulate stats from 3072 documents (11068746 virtual) 2018-03-26 17:11:26,057 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11260235 virtual) 2018-03-26 17:11:26,761 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11440306 virtual) 2018-03-26 17:11:28,201 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11862861 virtual) 2018-03-26 17:11:30,877 : INFO : 52 batches submitted to accumulate stats from 3328 documents (12180790 virtual) 2018-03-26 17:11:32,083 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12436827 virtual) 2018-03-26 17:11:32,241 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12694711 virtual) 2018-03-26 17:11:34,266 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12907287 virtual) 2018-03-26 17:11:35,114 : INFO : 56 batches submitted to accumulate stats from 3584 documents (13051609 virtual) 2018-03-26 17:11:41,777 : INFO : serializing accumulator to return to master... 2018-03-26 17:11:41,833 : INFO : accumulator serialized 2018-03-26 17:11:42,590 : INFO : serializing accumulator to return to master... 2018-03-26 17:11:42,596 : INFO : accumulator serialized 2018-03-26 17:11:42,969 : INFO : serializing accumulator to return to master... 2018-03-26 17:11:42,975 : INFO : accumulator serialized 2018-03-26 17:11:43,088 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:11:43,137 : INFO : accumulated word occurrence stats for 13051679 virtual documents 2018-03-26 17:11:43,470 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_15 2018-03-26 17:11:43,502 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_15.expElogbeta.npy with mmap=None 2018-03-26 17:11:43,544 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:11:43,545 : INFO : setting ignored attribute id2word to None 2018-03-26 17:11:43,545 : INFO : setting ignored attribute state to None 2018-03-26 17:11:43,546 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_15 2018-03-26 17:11:43,547 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_15.state 2018-03-26 17:11:43,604 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_15.state 2018-03-26 17:11:43,639 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:11:43,756 : INFO : 1 batches submitted to accumulate stats from 64 documents (231281 virtual) 2018-03-26 17:11:43,866 : INFO : 2 batches submitted to accumulate stats from 128 documents (559453 virtual) 2018-03-26 17:11:43,970 : INFO : 3 batches submitted to accumulate stats from 192 documents (769174 virtual) 2018-03-26 17:11:44,099 : INFO : 4 batches submitted to accumulate stats from 256 documents (990258 virtual) 2018-03-26 17:11:44,232 : INFO : 5 batches submitted to accumulate stats from 320 documents (1206756 virtual) 2018-03-26 17:11:44,346 : INFO : 6 batches submitted to accumulate stats from 384 documents (1420825 virtual) 2018-03-26 17:11:47,967 : INFO : 7 batches submitted to accumulate stats from 448 documents (1656938 virtual) 2018-03-26 17:11:48,473 : INFO : 8 batches submitted to accumulate stats from 512 documents (1850171 virtual) 2018-03-26 17:11:50,609 : INFO : 9 batches submitted to accumulate stats from 576 documents (2136550 virtual) 2018-03-26 17:11:52,335 : INFO : 10 batches submitted to accumulate stats from 640 documents (2339046 virtual) 2018-03-26 17:11:52,647 : INFO : 11 batches submitted to accumulate stats from 704 documents (2530358 virtual) 2018-03-26 17:11:54,913 : INFO : 12 batches submitted to accumulate stats from 768 documents (2711560 virtual) 2018-03-26 17:11:56,335 : INFO : 13 batches submitted to accumulate stats from 832 documents (2953166 virtual) 2018-03-26 17:11:56,795 : INFO : 14 batches submitted to accumulate stats from 896 documents (3167467 virtual) 2018-03-26 17:12:00,234 : INFO : 15 batches submitted to accumulate stats from 960 documents (3445606 virtual) 2018-03-26 17:12:00,474 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3666940 virtual) 2018-03-26 17:12:00,622 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3914084 virtual) 2018-03-26 17:12:03,704 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4156793 virtual) 2018-03-26 17:12:04,848 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4374402 virtual) 2018-03-26 17:12:05,364 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4598191 virtual) 2018-03-26 17:12:09,099 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4816189 virtual) 2018-03-26 17:12:09,274 : INFO : 22 batches submitted to accumulate stats from 1408 documents (5007308 virtual) 2018-03-26 17:12:10,304 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5242845 virtual) 2018-03-26 17:12:13,730 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5473175 virtual) 2018-03-26 17:12:13,979 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5678487 virtual) 2018-03-26 17:12:14,779 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5909047 virtual) 2018-03-26 17:12:17,690 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6144568 virtual) 2018-03-26 17:12:18,180 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6348383 virtual) 2018-03-26 17:12:19,540 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6576919 virtual) 2018-03-26 17:12:22,131 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6799947 virtual) 2018-03-26 17:12:22,350 : INFO : 31 batches submitted to accumulate stats from 1984 documents (7003398 virtual) 2018-03-26 17:12:24,154 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7229746 virtual) 2018-03-26 17:12:26,413 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7515611 virtual) 2018-03-26 17:12:26,872 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7734521 virtual) 2018-03-26 17:12:28,516 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7925119 virtual) 2018-03-26 17:12:30,699 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8167784 virtual) 2018-03-26 17:12:30,986 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8349050 virtual) 2018-03-26 17:12:33,004 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8603682 virtual) 2018-03-26 17:12:35,355 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8818932 virtual) 2018-03-26 17:12:36,179 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9052264 virtual) 2018-03-26 17:12:36,792 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9229637 virtual) 2018-03-26 17:12:39,915 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9461829 virtual) 2018-03-26 17:12:40,218 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9791136 virtual) 2018-03-26 17:12:41,713 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9991929 virtual) 2018-03-26 17:12:44,144 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10174425 virtual) 2018-03-26 17:12:44,831 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10491649 virtual) 2018-03-26 17:12:45,231 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10764539 virtual) 2018-03-26 17:12:48,822 : INFO : 48 batches submitted to accumulate stats from 3072 documents (11012060 virtual) 2018-03-26 17:12:49,226 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11188647 virtual) 2018-03-26 17:12:51,315 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11383859 virtual) 2018-03-26 17:12:52,522 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11736222 virtual) 2018-03-26 17:12:55,367 : INFO : 52 batches submitted to accumulate stats from 3328 documents (12107946 virtual) 2018-03-26 17:12:56,847 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12358484 virtual) 2018-03-26 17:12:57,575 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12610690 virtual) 2018-03-26 17:12:58,857 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12799716 virtual) 2018-03-26 17:13:00,767 : INFO : 56 batches submitted to accumulate stats from 3584 documents (13052258 virtual) 2018-03-26 17:13:09,035 : INFO : serializing accumulator to return to master... 2018-03-26 17:13:09,042 : INFO : accumulator serialized 2018-03-26 17:13:09,332 : INFO : serializing accumulator to return to master... 2018-03-26 17:13:09,337 : INFO : accumulator serialized 2018-03-26 17:13:10,495 : INFO : serializing accumulator to return to master... 2018-03-26 17:13:10,501 : INFO : accumulator serialized 2018-03-26 17:13:10,672 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:13:10,766 : INFO : accumulated word occurrence stats for 13052341 virtual documents 2018-03-26 17:13:11,231 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_20 2018-03-26 17:13:11,245 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_20.expElogbeta.npy with mmap=None 2018-03-26 17:13:11,264 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:13:11,265 : INFO : setting ignored attribute id2word to None 2018-03-26 17:13:11,265 : INFO : setting ignored attribute state to None 2018-03-26 17:13:11,266 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_20 2018-03-26 17:13:11,266 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_20.state 2018-03-26 17:13:11,325 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_20.state 2018-03-26 17:13:11,375 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:13:11,493 : INFO : 1 batches submitted to accumulate stats from 64 documents (230352 virtual) 2018-03-26 17:13:11,603 : INFO : 2 batches submitted to accumulate stats from 128 documents (559397 virtual) 2018-03-26 17:13:11,695 : INFO : 3 batches submitted to accumulate stats from 192 documents (764119 virtual) 2018-03-26 17:13:11,819 : INFO : 4 batches submitted to accumulate stats from 256 documents (984084 virtual) 2018-03-26 17:13:11,960 : INFO : 5 batches submitted to accumulate stats from 320 documents (1202061 virtual) 2018-03-26 17:13:12,079 : INFO : 6 batches submitted to accumulate stats from 384 documents (1409589 virtual) 2018-03-26 17:13:15,958 : INFO : 7 batches submitted to accumulate stats from 448 documents (1643855 virtual) 2018-03-26 17:13:16,380 : INFO : 8 batches submitted to accumulate stats from 512 documents (1841456 virtual) 2018-03-26 17:13:18,754 : INFO : 9 batches submitted to accumulate stats from 576 documents (2126565 virtual) 2018-03-26 17:13:20,676 : INFO : 10 batches submitted to accumulate stats from 640 documents (2333631 virtual) 2018-03-26 17:13:21,063 : INFO : 11 batches submitted to accumulate stats from 704 documents (2530202 virtual) 2018-03-26 17:13:23,086 : INFO : 12 batches submitted to accumulate stats from 768 documents (2708934 virtual) 2018-03-26 17:13:25,092 : INFO : 13 batches submitted to accumulate stats from 832 documents (2949095 virtual) 2018-03-26 17:13:25,500 : INFO : 14 batches submitted to accumulate stats from 896 documents (3166182 virtual) 2018-03-26 17:13:29,178 : INFO : 15 batches submitted to accumulate stats from 960 documents (3437861 virtual) 2018-03-26 17:13:29,299 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3655177 virtual) 2018-03-26 17:13:29,510 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3907894 virtual) 2018-03-26 17:13:32,721 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4152711 virtual) 2018-03-26 17:13:34,086 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4362842 virtual) 2018-03-26 17:13:34,551 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4594681 virtual) 2018-03-26 17:13:38,392 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4810487 virtual) 2018-03-26 17:13:38,684 : INFO : 22 batches submitted to accumulate stats from 1408 documents (5002194 virtual) 2018-03-26 17:13:39,899 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5235575 virtual) 2018-03-26 17:13:42,985 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5459548 virtual) 2018-03-26 17:13:43,530 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5666841 virtual) 2018-03-26 17:13:44,972 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5895843 virtual) 2018-03-26 17:13:47,322 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6121967 virtual) 2018-03-26 17:13:47,487 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6343148 virtual) 2018-03-26 17:13:49,933 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6569836 virtual) 2018-03-26 17:13:51,969 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6789309 virtual) 2018-03-26 17:13:52,105 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6994811 virtual) 2018-03-26 17:13:54,626 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7229433 virtual) 2018-03-26 17:13:56,619 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7500323 virtual) 2018-03-26 17:13:56,792 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7721342 virtual) 2018-03-26 17:13:59,274 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7919683 virtual) 2018-03-26 17:14:01,165 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8158778 virtual) 2018-03-26 17:14:01,739 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8345397 virtual) 2018-03-26 17:14:04,916 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8586213 virtual) 2018-03-26 17:14:06,953 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8807825 virtual) 2018-03-26 17:14:07,377 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9040425 virtual) 2018-03-26 17:14:09,225 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9218460 virtual) 2018-03-26 17:14:11,321 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9448725 virtual) 2018-03-26 17:14:11,978 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9775492 virtual) 2018-03-26 17:14:14,094 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9974446 virtual) 2018-03-26 17:14:16,137 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10169686 virtual) 2018-03-26 17:14:16,790 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10469976 virtual) 2018-03-26 17:14:17,836 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10740142 virtual) 2018-03-26 17:14:21,078 : INFO : 48 batches submitted to accumulate stats from 3072 documents (11001644 virtual) 2018-03-26 17:14:22,044 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11164024 virtual) 2018-03-26 17:14:23,420 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11359038 virtual) 2018-03-26 17:14:25,206 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11637933 virtual) 2018-03-26 17:14:28,361 : INFO : 52 batches submitted to accumulate stats from 3328 documents (12096713 virtual) 2018-03-26 17:14:28,809 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12332461 virtual) 2018-03-26 17:14:30,841 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12582544 virtual) 2018-03-26 17:14:31,793 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12774562 virtual) 2018-03-26 17:14:32,948 : INFO : 56 batches submitted to accumulate stats from 3584 documents (13027501 virtual) 2018-03-26 17:14:36,027 : INFO : 57 batches submitted to accumulate stats from 3648 documents (13052301 virtual) 2018-03-26 17:14:41,619 : INFO : serializing accumulator to return to master... 2018-03-26 17:14:41,753 : INFO : serializing accumulator to return to master... 2018-03-26 17:14:41,626 : INFO : accumulator serialized 2018-03-26 17:14:41,764 : INFO : accumulator serialized 2018-03-26 17:14:44,712 : INFO : serializing accumulator to return to master... 2018-03-26 17:14:44,718 : INFO : accumulator serialized 2018-03-26 17:14:44,902 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:14:45,046 : INFO : accumulated word occurrence stats for 13052389 virtual documents 2018-03-26 17:14:45,667 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_25 2018-03-26 17:14:45,676 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-26 17:14:45,691 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:14:45,691 : INFO : setting ignored attribute id2word to None 2018-03-26 17:14:45,692 : INFO : setting ignored attribute state to None 2018-03-26 17:14:45,693 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_25 2018-03-26 17:14:45,693 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_25.state 2018-03-26 17:14:45,757 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_25.state 2018-03-26 17:14:45,800 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:14:45,918 : INFO : 1 batches submitted to accumulate stats from 64 documents (230352 virtual) 2018-03-26 17:14:46,030 : INFO : 2 batches submitted to accumulate stats from 128 documents (559397 virtual) 2018-03-26 17:14:46,133 : INFO : 3 batches submitted to accumulate stats from 192 documents (764119 virtual) 2018-03-26 17:14:46,260 : INFO : 4 batches submitted to accumulate stats from 256 documents (984084 virtual) 2018-03-26 17:14:46,393 : INFO : 5 batches submitted to accumulate stats from 320 documents (1202061 virtual) 2018-03-26 17:14:46,511 : INFO : 6 batches submitted to accumulate stats from 384 documents (1409589 virtual) 2018-03-26 17:14:50,490 : INFO : 7 batches submitted to accumulate stats from 448 documents (1643855 virtual) 2018-03-26 17:14:51,008 : INFO : 8 batches submitted to accumulate stats from 512 documents (1841456 virtual) 2018-03-26 17:14:53,446 : INFO : 9 batches submitted to accumulate stats from 576 documents (2126565 virtual) 2018-03-26 17:14:55,455 : INFO : 10 batches submitted to accumulate stats from 640 documents (2333631 virtual) 2018-03-26 17:14:55,798 : INFO : 11 batches submitted to accumulate stats from 704 documents (2530202 virtual) 2018-03-26 17:14:58,087 : INFO : 12 batches submitted to accumulate stats from 768 documents (2708934 virtual) 2018-03-26 17:14:59,987 : INFO : 13 batches submitted to accumulate stats from 832 documents (2949095 virtual) 2018-03-26 17:15:00,283 : INFO : 14 batches submitted to accumulate stats from 896 documents (3161770 virtual) 2018-03-26 17:15:04,238 : INFO : 15 batches submitted to accumulate stats from 960 documents (3433595 virtual) 2018-03-26 17:15:04,496 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3653166 virtual) 2018-03-26 17:15:04,640 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3888639 virtual) 2018-03-26 17:15:08,024 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4145210 virtual) 2018-03-26 17:15:09,057 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4352524 virtual) 2018-03-26 17:15:09,963 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4581199 virtual) 2018-03-26 17:15:13,835 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4799816 virtual) 2018-03-26 17:15:14,106 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4995620 virtual) 2018-03-26 17:15:14,994 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5221976 virtual) 2018-03-26 17:15:18,577 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5448528 virtual) 2018-03-26 17:15:19,456 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5642426 virtual) 2018-03-26 17:15:20,043 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5885523 virtual) 2018-03-26 17:15:23,338 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6115460 virtual) 2018-03-26 17:15:23,731 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6337698 virtual) 2018-03-26 17:15:24,880 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6549463 virtual) 2018-03-26 17:15:27,994 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6759897 virtual) 2018-03-26 17:15:28,176 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6975998 virtual) 2018-03-26 17:15:30,088 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7213305 virtual) 2018-03-26 17:15:32,972 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7479801 virtual) 2018-03-26 17:15:33,123 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7713294 virtual) 2018-03-26 17:15:34,578 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7905043 virtual) 2018-03-26 17:15:37,640 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8136209 virtual) 2018-03-26 17:15:37,750 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8321935 virtual) 2018-03-26 17:15:39,662 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8578620 virtual) 2018-03-26 17:15:42,845 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8795567 virtual) 2018-03-26 17:15:43,413 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9024677 virtual) 2018-03-26 17:15:43,790 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9208981 virtual) 2018-03-26 17:15:47,473 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9427891 virtual) 2018-03-26 17:15:47,954 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9766159 virtual) 2018-03-26 17:15:49,340 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9952800 virtual) 2018-03-26 17:15:52,303 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10158712 virtual) 2018-03-26 17:15:52,922 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10427573 virtual) 2018-03-26 17:15:53,373 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10723068 virtual) 2018-03-26 17:15:57,059 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10945449 virtual) 2018-03-26 17:15:57,467 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11144238 virtual) 2018-03-26 17:16:00,473 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11348032 virtual) 2018-03-26 17:16:01,588 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11586244 virtual) 2018-03-26 17:16:03,448 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11943643 virtual) 2018-03-26 17:16:06,300 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12293656 virtual) 2018-03-26 17:16:07,019 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12543477 virtual) 2018-03-26 17:16:07,691 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12732815 virtual) 2018-03-26 17:16:10,767 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12999301 virtual) 2018-03-26 17:16:11,467 : INFO : 57 batches submitted to accumulate stats from 3648 documents (13052325 virtual) 2018-03-26 17:16:19,163 : INFO : serializing accumulator to return to master... 2018-03-26 17:16:19,170 : INFO : accumulator serialized 2018-03-26 17:16:19,508 : INFO : serializing accumulator to return to master... 2018-03-26 17:16:19,514 : INFO : accumulator serialized 2018-03-26 17:16:21,394 : INFO : serializing accumulator to return to master... 2018-03-26 17:16:21,400 : INFO : accumulator serialized 2018-03-26 17:16:21,633 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:16:21,823 : INFO : accumulated word occurrence stats for 13052417 virtual documents 2018-03-26 17:16:22,578 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_30 2018-03-26 17:16:22,612 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-26 17:16:22,718 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:16:22,719 : INFO : setting ignored attribute id2word to None 2018-03-26 17:16:22,719 : INFO : setting ignored attribute state to None 2018-03-26 17:16:22,720 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_30 2018-03-26 17:16:22,720 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_30.state 2018-03-26 17:16:22,879 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_30.state 2018-03-26 17:16:22,938 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:16:23,058 : INFO : 1 batches submitted to accumulate stats from 64 documents (230352 virtual) 2018-03-26 17:16:23,171 : INFO : 2 batches submitted to accumulate stats from 128 documents (559397 virtual) 2018-03-26 17:16:23,270 : INFO : 3 batches submitted to accumulate stats from 192 documents (764119 virtual) 2018-03-26 17:16:23,402 : INFO : 4 batches submitted to accumulate stats from 256 documents (984084 virtual) 2018-03-26 17:16:23,537 : INFO : 5 batches submitted to accumulate stats from 320 documents (1202061 virtual) 2018-03-26 17:16:23,660 : INFO : 6 batches submitted to accumulate stats from 384 documents (1404995 virtual) 2018-03-26 17:16:27,955 : INFO : 7 batches submitted to accumulate stats from 448 documents (1636823 virtual) 2018-03-26 17:16:28,464 : INFO : 8 batches submitted to accumulate stats from 512 documents (1841427 virtual) 2018-03-26 17:16:30,794 : INFO : 9 batches submitted to accumulate stats from 576 documents (2125656 virtual) 2018-03-26 17:16:32,891 : INFO : 10 batches submitted to accumulate stats from 640 documents (2332404 virtual) 2018-03-26 17:16:33,441 : INFO : 11 batches submitted to accumulate stats from 704 documents (2529581 virtual) 2018-03-26 17:16:35,375 : INFO : 12 batches submitted to accumulate stats from 768 documents (2703754 virtual) 2018-03-26 17:16:37,864 : INFO : 13 batches submitted to accumulate stats from 832 documents (2944401 virtual) 2018-03-26 17:16:38,002 : INFO : 14 batches submitted to accumulate stats from 896 documents (3161780 virtual) 2018-03-26 17:16:41,736 : INFO : 15 batches submitted to accumulate stats from 960 documents (3433605 virtual) 2018-03-26 17:16:42,474 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3655084 virtual) 2018-03-26 17:16:42,692 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3888646 virtual) 2018-03-26 17:16:45,724 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4145217 virtual) 2018-03-26 17:16:47,962 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4352531 virtual) 2018-03-26 17:16:48,246 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4581206 virtual) 2018-03-26 17:16:51,859 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4799823 virtual) 2018-03-26 17:16:53,036 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4995627 virtual) 2018-03-26 17:16:53,554 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5221983 virtual) 2018-03-26 17:16:57,474 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5448535 virtual) 2018-03-26 17:16:57,765 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5642433 virtual) 2018-03-26 17:16:58,943 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5885530 virtual) 2018-03-26 17:17:02,177 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6115467 virtual) 2018-03-26 17:17:02,312 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6337705 virtual) 2018-03-26 17:17:03,956 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6544919 virtual) 2018-03-26 17:17:06,795 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6759893 virtual) 2018-03-26 17:17:07,245 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6975994 virtual) 2018-03-26 17:17:09,428 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7209506 virtual) 2018-03-26 17:17:11,980 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7475553 virtual) 2018-03-26 17:17:12,174 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7709220 virtual) 2018-03-26 17:17:14,026 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7903239 virtual) 2018-03-26 17:17:16,951 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8136117 virtual) 2018-03-26 17:17:17,065 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8321838 virtual) 2018-03-26 17:17:19,308 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8567438 virtual) 2018-03-26 17:17:22,283 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8790276 virtual) 2018-03-26 17:17:23,102 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9009423 virtual) 2018-03-26 17:17:23,735 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9200375 virtual) 2018-03-26 17:17:27,343 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9427888 virtual) 2018-03-26 17:17:27,546 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9766156 virtual) 2018-03-26 17:17:29,186 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9952797 virtual) 2018-03-26 17:17:32,326 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10158709 virtual) 2018-03-26 17:17:32,485 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10427570 virtual) 2018-03-26 17:17:33,399 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10723065 virtual) 2018-03-26 17:17:37,419 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10949652 virtual) 2018-03-26 17:17:37,568 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11149261 virtual) 2018-03-26 17:17:40,220 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11349591 virtual) 2018-03-26 17:17:41,937 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11595569 virtual) 2018-03-26 17:17:43,519 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11943663 virtual) 2018-03-26 17:17:46,744 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12300259 virtual) 2018-03-26 17:17:47,078 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12552436 virtual) 2018-03-26 17:17:47,847 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12737406 virtual) 2018-03-26 17:17:51,313 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12999367 virtual) 2018-03-26 17:17:51,743 : INFO : 57 batches submitted to accumulate stats from 3648 documents (13052317 virtual) 2018-03-26 17:17:59,983 : INFO : serializing accumulator to return to master... 2018-03-26 17:18:00,018 : INFO : serializing accumulator to return to master... 2018-03-26 17:18:00,045 : INFO : accumulator serialized 2018-03-26 17:18:00,045 : INFO : accumulator serialized 2018-03-26 17:18:01,968 : INFO : serializing accumulator to return to master... 2018-03-26 17:18:01,973 : INFO : accumulator serialized 2018-03-26 17:18:02,240 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:18:02,483 : INFO : accumulated word occurrence stats for 13052412 virtual documents 2018-03-26 17:18:03,370 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_35 2018-03-26 17:18:03,398 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_35.expElogbeta.npy with mmap=None 2018-03-26 17:18:03,505 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:18:03,506 : INFO : setting ignored attribute id2word to None 2018-03-26 17:18:03,506 : INFO : setting ignored attribute state to None 2018-03-26 17:18:03,507 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_35 2018-03-26 17:18:03,507 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_35.state 2018-03-26 17:18:03,749 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_35.state 2018-03-26 17:18:03,816 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:18:03,937 : INFO : 1 batches submitted to accumulate stats from 64 documents (230352 virtual) 2018-03-26 17:18:04,066 : INFO : 2 batches submitted to accumulate stats from 128 documents (559397 virtual) 2018-03-26 17:18:04,183 : INFO : 3 batches submitted to accumulate stats from 192 documents (769182 virtual) 2018-03-26 17:18:04,312 : INFO : 4 batches submitted to accumulate stats from 256 documents (984088 virtual) 2018-03-26 17:18:04,438 : INFO : 5 batches submitted to accumulate stats from 320 documents (1206717 virtual) 2018-03-26 17:18:04,552 : INFO : 6 batches submitted to accumulate stats from 384 documents (1409603 virtual) 2018-03-26 17:18:09,057 : INFO : 7 batches submitted to accumulate stats from 448 documents (1643869 virtual) 2018-03-26 17:18:09,482 : INFO : 8 batches submitted to accumulate stats from 512 documents (1841470 virtual) 2018-03-26 17:18:11,969 : INFO : 9 batches submitted to accumulate stats from 576 documents (2126579 virtual) 2018-03-26 17:18:14,214 : INFO : 10 batches submitted to accumulate stats from 640 documents (2333645 virtual) 2018-03-26 17:18:14,752 : INFO : 11 batches submitted to accumulate stats from 704 documents (2530216 virtual) 2018-03-26 17:18:16,745 : INFO : 12 batches submitted to accumulate stats from 768 documents (2708948 virtual) 2018-03-26 17:18:19,393 : INFO : 13 batches submitted to accumulate stats from 832 documents (2949109 virtual) 2018-03-26 17:18:19,523 : INFO : 14 batches submitted to accumulate stats from 896 documents (3161784 virtual) 2018-03-26 17:18:23,657 : INFO : 15 batches submitted to accumulate stats from 960 documents (3433609 virtual) 2018-03-26 17:18:23,949 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3653180 virtual) 2018-03-26 17:18:24,144 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3888555 virtual) 2018-03-26 17:18:27,689 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4140500 virtual) 2018-03-26 17:18:29,139 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4348104 virtual) 2018-03-26 17:18:29,811 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4581142 virtual) 2018-03-26 17:18:34,130 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4794963 virtual) 2018-03-26 17:18:34,371 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4995635 virtual) 2018-03-26 17:18:35,281 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5214846 virtual) 2018-03-26 17:18:39,289 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5445551 virtual) 2018-03-26 17:18:40,248 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5635855 virtual) 2018-03-26 17:18:40,683 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5884577 virtual) 2018-03-26 17:18:44,275 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6111350 virtual) 2018-03-26 17:18:44,939 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6337664 virtual) 2018-03-26 17:18:45,730 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6541797 virtual) 2018-03-26 17:18:49,314 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6752412 virtual) 2018-03-26 17:18:49,673 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6969371 virtual) 2018-03-26 17:18:51,397 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7199112 virtual) 2018-03-26 17:18:54,716 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7467096 virtual) 2018-03-26 17:18:54,862 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7703850 virtual) 2018-03-26 17:18:56,135 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7884462 virtual) 2018-03-26 17:18:59,648 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8130174 virtual) 2018-03-26 17:18:59,893 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8303486 virtual) 2018-03-26 17:19:01,511 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8559218 virtual) 2018-03-26 17:19:05,343 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8778782 virtual) 2018-03-26 17:19:05,774 : INFO : 40 batches submitted to accumulate stats from 2560 documents (8998046 virtual) 2018-03-26 17:19:05,935 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9183713 virtual) 2018-03-26 17:19:09,816 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9416643 virtual) 2018-03-26 17:19:11,123 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9759781 virtual) 2018-03-26 17:19:11,785 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9945666 virtual) 2018-03-26 17:19:14,971 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10155851 virtual) 2018-03-26 17:19:16,065 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10408012 virtual) 2018-03-26 17:19:16,445 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10711789 virtual) 2018-03-26 17:19:20,418 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10924373 virtual) 2018-03-26 17:19:20,827 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11129054 virtual) 2018-03-26 17:19:24,576 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11339944 virtual) 2018-03-26 17:19:25,748 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11573381 virtual) 2018-03-26 17:19:27,406 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11928446 virtual) 2018-03-26 17:19:30,798 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12292558 virtual) 2018-03-26 17:19:31,682 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12525322 virtual) 2018-03-26 17:19:32,129 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12727513 virtual) 2018-03-26 17:19:35,946 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12974377 virtual) 2018-03-26 17:19:36,421 : INFO : 57 batches submitted to accumulate stats from 3648 documents (13052341 virtual) 2018-03-26 17:19:44,769 : INFO : serializing accumulator to return to master... 2018-03-26 17:19:44,875 : INFO : accumulator serialized 2018-03-26 17:19:45,608 : INFO : serializing accumulator to return to master... 2018-03-26 17:19:45,615 : INFO : accumulator serialized 2018-03-26 17:19:46,486 : INFO : serializing accumulator to return to master... 2018-03-26 17:19:46,491 : INFO : accumulator serialized 2018-03-26 17:19:46,802 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:19:47,115 : INFO : accumulated word occurrence stats for 13052433 virtual documents 2018-03-26 17:19:48,174 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_40 2018-03-26 17:19:48,206 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-26 17:19:48,328 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:19:48,329 : INFO : setting ignored attribute id2word to None 2018-03-26 17:19:48,330 : INFO : setting ignored attribute state to None 2018-03-26 17:19:48,330 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_40 2018-03-26 17:19:48,331 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_40.state 2018-03-26 17:19:48,685 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_40.state 2018-03-26 17:19:48,755 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:19:48,879 : INFO : 1 batches submitted to accumulate stats from 64 documents (230352 virtual) 2018-03-26 17:19:49,005 : INFO : 2 batches submitted to accumulate stats from 128 documents (559397 virtual) 2018-03-26 17:19:49,093 : INFO : 3 batches submitted to accumulate stats from 192 documents (769182 virtual) 2018-03-26 17:19:49,219 : INFO : 4 batches submitted to accumulate stats from 256 documents (984088 virtual) 2018-03-26 17:19:49,353 : INFO : 5 batches submitted to accumulate stats from 320 documents (1206717 virtual) 2018-03-26 17:19:49,474 : INFO : 6 batches submitted to accumulate stats from 384 documents (1409603 virtual) 2018-03-26 17:19:54,170 : INFO : 7 batches submitted to accumulate stats from 448 documents (1643869 virtual) 2018-03-26 17:19:54,463 : INFO : 8 batches submitted to accumulate stats from 512 documents (1841446 virtual) 2018-03-26 17:19:57,141 : INFO : 9 batches submitted to accumulate stats from 576 documents (2125675 virtual) 2018-03-26 17:19:59,265 : INFO : 10 batches submitted to accumulate stats from 640 documents (2332423 virtual) 2018-03-26 17:19:59,791 : INFO : 11 batches submitted to accumulate stats from 704 documents (2529600 virtual) 2018-03-26 17:20:01,932 : INFO : 12 batches submitted to accumulate stats from 768 documents (2703773 virtual) 2018-03-26 17:20:04,247 : INFO : 13 batches submitted to accumulate stats from 832 documents (2944420 virtual) 2018-03-26 17:20:04,913 : INFO : 14 batches submitted to accumulate stats from 896 documents (3158639 virtual) 2018-03-26 17:20:08,642 : INFO : 15 batches submitted to accumulate stats from 960 documents (3433578 virtual) 2018-03-26 17:20:08,959 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3653065 virtual) 2018-03-26 17:20:09,562 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3888572 virtual) 2018-03-26 17:20:12,716 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4140517 virtual) 2018-03-26 17:20:14,546 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4348121 virtual) 2018-03-26 17:20:14,759 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4581229 virtual) 2018-03-26 17:20:19,361 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4799846 virtual) 2018-03-26 17:20:19,746 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4996990 virtual) 2018-03-26 17:20:20,274 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5222132 virtual) 2018-03-26 17:20:24,837 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5451093 virtual) 2018-03-26 17:20:25,407 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5653400 virtual) 2018-03-26 17:20:25,940 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5885704 virtual) 2018-03-26 17:20:29,952 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6115578 virtual) 2018-03-26 17:20:30,223 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6337796 virtual) 2018-03-26 17:20:31,461 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6549491 virtual) 2018-03-26 17:20:35,340 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6766759 virtual) 2018-03-26 17:20:35,588 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6979355 virtual) 2018-03-26 17:20:36,765 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7213325 virtual) 2018-03-26 17:20:40,829 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7479826 virtual) 2018-03-26 17:20:40,967 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7713319 virtual) 2018-03-26 17:20:41,820 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7905068 virtual) 2018-03-26 17:20:45,845 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8136234 virtual) 2018-03-26 17:20:45,954 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8321960 virtual) 2018-03-26 17:20:47,665 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8578645 virtual) 2018-03-26 17:20:51,389 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8795592 virtual) 2018-03-26 17:20:52,090 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9024702 virtual) 2018-03-26 17:20:52,554 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9209006 virtual) 2018-03-26 17:20:56,613 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9427916 virtual) 2018-03-26 17:20:56,893 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9766184 virtual) 2018-03-26 17:20:58,644 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9952825 virtual) 2018-03-26 17:21:01,801 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10158737 virtual) 2018-03-26 17:21:02,514 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10427598 virtual) 2018-03-26 17:21:03,130 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10723093 virtual) 2018-03-26 17:21:07,113 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10945474 virtual) 2018-03-26 17:21:07,830 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11144263 virtual) 2018-03-26 17:21:10,536 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11348001 virtual) 2018-03-26 17:21:12,003 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11586162 virtual) 2018-03-26 17:21:14,428 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11936645 virtual) 2018-03-26 17:21:17,438 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12293577 virtual) 2018-03-26 17:21:17,741 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12540152 virtual) 2018-03-26 17:21:18,989 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12732684 virtual) 2018-03-26 17:21:22,612 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12994901 virtual) 2018-03-26 17:21:22,893 : INFO : 57 batches submitted to accumulate stats from 3648 documents (13052346 virtual) 2018-03-26 17:21:32,539 : INFO : serializing accumulator to return to master... 2018-03-26 17:21:32,687 : INFO : serializing accumulator to return to master... 2018-03-26 17:21:32,562 : INFO : accumulator serialized 2018-03-26 17:21:32,693 : INFO : accumulator serialized 2018-03-26 17:21:34,597 : INFO : serializing accumulator to return to master... 2018-03-26 17:21:34,604 : INFO : accumulator serialized 2018-03-26 17:21:34,986 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:21:35,340 : INFO : accumulated word occurrence stats for 13052439 virtual documents 2018-03-26 17:21:36,538 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_50 2018-03-26 17:21:36,572 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_50.expElogbeta.npy with mmap=None 2018-03-26 17:21:36,719 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:21:36,719 : INFO : setting ignored attribute id2word to None 2018-03-26 17:21:36,720 : INFO : setting ignored attribute state to None 2018-03-26 17:21:36,720 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_50 2018-03-26 17:21:36,721 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_50.state 2018-03-26 17:21:37,001 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_50.state 2018-03-26 17:21:37,077 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:21:37,201 : INFO : 1 batches submitted to accumulate stats from 64 documents (230352 virtual) 2018-03-26 17:21:37,339 : INFO : 2 batches submitted to accumulate stats from 128 documents (559397 virtual) 2018-03-26 17:21:37,452 : INFO : 3 batches submitted to accumulate stats from 192 documents (769182 virtual) 2018-03-26 17:21:37,578 : INFO : 4 batches submitted to accumulate stats from 256 documents (984088 virtual) 2018-03-26 17:21:37,716 : INFO : 5 batches submitted to accumulate stats from 320 documents (1206717 virtual) 2018-03-26 17:21:37,843 : INFO : 6 batches submitted to accumulate stats from 384 documents (1409603 virtual) 2018-03-26 17:21:42,822 : INFO : 7 batches submitted to accumulate stats from 448 documents (1643869 virtual) 2018-03-26 17:21:43,068 : INFO : 8 batches submitted to accumulate stats from 512 documents (1841446 virtual) 2018-03-26 17:21:46,529 : INFO : 9 batches submitted to accumulate stats from 576 documents (2125675 virtual) 2018-03-26 17:21:50,022 : INFO : 10 batches submitted to accumulate stats from 640 documents (2323488 virtual) 2018-03-26 17:21:50,172 : INFO : 11 batches submitted to accumulate stats from 704 documents (2529499 virtual) 2018-03-26 17:21:53,169 : INFO : 12 batches submitted to accumulate stats from 768 documents (2702828 virtual) 2018-03-26 17:21:56,620 : INFO : 13 batches submitted to accumulate stats from 832 documents (2944286 virtual) 2018-03-26 17:21:57,801 : INFO : 14 batches submitted to accumulate stats from 896 documents (3155962 virtual) 2018-03-26 17:22:02,597 : INFO : 15 batches submitted to accumulate stats from 960 documents (3426003 virtual) 2018-03-26 17:22:03,232 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3652970 virtual) 2018-03-26 17:22:04,968 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3888435 virtual) 2018-03-26 17:22:08,523 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4131164 virtual) 2018-03-26 17:22:11,632 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4346215 virtual) 2018-03-26 17:22:12,136 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4569973 virtual) 2018-03-26 17:22:17,444 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4789055 virtual) 2018-03-26 17:22:19,453 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4986968 virtual) 2018-03-26 17:22:19,766 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5201731 virtual) 2018-03-26 17:22:25,469 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5431479 virtual) 2018-03-26 17:22:26,729 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5620766 virtual) 2018-03-26 17:22:27,467 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5878039 virtual) 2018-03-26 17:22:32,328 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6111135 virtual) 2018-03-26 17:22:33,397 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6324451 virtual) 2018-03-26 17:22:34,643 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6537590 virtual) 2018-03-26 17:22:39,663 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6723312 virtual) 2018-03-26 17:22:39,925 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6958421 virtual) 2018-03-26 17:22:43,175 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7179000 virtual) 2018-03-26 17:22:46,838 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7456322 virtual) 2018-03-26 17:22:47,536 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7679757 virtual) 2018-03-26 17:22:50,289 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7873432 virtual) 2018-03-26 17:22:53,073 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8113009 virtual) 2018-03-26 17:22:55,459 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8275586 virtual) 2018-03-26 17:22:57,553 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8536239 virtual) 2018-03-26 17:23:02,520 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8747997 virtual) 2018-03-26 17:23:02,702 : INFO : 40 batches submitted to accumulate stats from 2560 documents (8969582 virtual) 2018-03-26 17:23:03,998 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9167439 virtual) 2018-03-26 17:23:08,062 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9387169 virtual) 2018-03-26 17:23:10,655 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9724487 virtual) 2018-03-26 17:23:12,520 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9919629 virtual) 2018-03-26 17:23:15,093 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10136908 virtual) 2018-03-26 17:23:18,388 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10376105 virtual) 2018-03-26 17:23:18,940 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10692493 virtual) 2018-03-26 17:23:22,101 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10914222 virtual) 2018-03-26 17:23:25,441 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11109471 virtual) 2018-03-26 17:23:29,436 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11328048 virtual) 2018-03-26 17:23:29,861 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11565278 virtual) 2018-03-26 17:23:33,327 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11907116 virtual) 2018-03-26 17:23:37,108 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12255679 virtual) 2018-03-26 17:23:39,708 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12493389 virtual) 2018-03-26 17:23:39,924 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12720845 virtual) 2018-03-26 17:23:44,656 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12940864 virtual) 2018-03-26 17:23:46,754 : INFO : 57 batches submitted to accumulate stats from 3648 documents (13052397 virtual) 2018-03-26 17:23:58,134 : INFO : serializing accumulator to return to master... 2018-03-26 17:23:58,192 : INFO : accumulator serialized 2018-03-26 17:23:59,360 : INFO : serializing accumulator to return to master... 2018-03-26 17:23:59,366 : INFO : accumulator serialized 2018-03-26 17:24:00,897 : INFO : serializing accumulator to return to master... 2018-03-26 17:24:00,906 : INFO : accumulator serialized 2018-03-26 17:24:01,444 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:24:02,027 : INFO : accumulated word occurrence stats for 13052498 virtual documents
In [17]:
tmrest_full_score['c_uci'] = tmrest_full_score.apply(lambda row: get_score(row.num_topic, dict_restricted, texts=text_tmrest, corpus=corpus_tmrest,model_dir='../src/tmrest/',
coherence='c_uci'),
axis=1)
2018-03-26 17:24:03,797 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_10 2018-03-26 17:24:03,861 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-26 17:24:03,936 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:24:03,936 : INFO : setting ignored attribute id2word to None 2018-03-26 17:24:03,937 : INFO : setting ignored attribute state to None 2018-03-26 17:24:03,937 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_10 2018-03-26 17:24:03,938 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_10.state 2018-03-26 17:24:04,014 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_10.state 2018-03-26 17:24:04,062 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:24:04,187 : INFO : 1 batches submitted to accumulate stats from 64 documents (234178 virtual) 2018-03-26 17:24:04,307 : INFO : 2 batches submitted to accumulate stats from 128 documents (561538 virtual) 2018-03-26 17:24:04,447 : INFO : 3 batches submitted to accumulate stats from 192 documents (772919 virtual) 2018-03-26 17:24:04,578 : INFO : 4 batches submitted to accumulate stats from 256 documents (1005918 virtual) 2018-03-26 17:24:04,693 : INFO : 5 batches submitted to accumulate stats from 320 documents (1212590 virtual) 2018-03-26 17:24:04,819 : INFO : 6 batches submitted to accumulate stats from 384 documents (1438804 virtual) 2018-03-26 17:24:09,486 : INFO : 7 batches submitted to accumulate stats from 448 documents (1660242 virtual) 2018-03-26 17:24:09,668 : INFO : 8 batches submitted to accumulate stats from 512 documents (1876192 virtual) 2018-03-26 17:24:12,058 : INFO : 9 batches submitted to accumulate stats from 576 documents (2148682 virtual) 2018-03-26 17:24:14,119 : INFO : 10 batches submitted to accumulate stats from 640 documents (2355798 virtual) 2018-03-26 17:24:14,406 : INFO : 11 batches submitted to accumulate stats from 704 documents (2534005 virtual) 2018-03-26 17:24:16,621 : INFO : 12 batches submitted to accumulate stats from 768 documents (2720756 virtual) 2018-03-26 17:24:18,246 : INFO : 13 batches submitted to accumulate stats from 832 documents (2971398 virtual) 2018-03-26 17:24:18,506 : INFO : 14 batches submitted to accumulate stats from 896 documents (3188709 virtual) 2018-03-26 17:24:21,712 : INFO : 15 batches submitted to accumulate stats from 960 documents (3468642 virtual) 2018-03-26 17:24:21,833 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3682140 virtual) 2018-03-26 17:24:22,204 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3929882 virtual) 2018-03-26 17:24:25,433 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4178101 virtual) 2018-03-26 17:24:26,640 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4394941 virtual) 2018-03-26 17:24:27,212 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4627886 virtual) 2018-03-26 17:24:31,207 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4843219 virtual) 2018-03-26 17:24:31,318 : INFO : 22 batches submitted to accumulate stats from 1408 documents (5046629 virtual) 2018-03-26 17:24:32,309 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5251646 virtual) 2018-03-26 17:24:35,751 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5498212 virtual) 2018-03-26 17:24:36,215 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5716104 virtual) 2018-03-26 17:24:37,111 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5980166 virtual) 2018-03-26 17:24:39,965 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6204717 virtual) 2018-03-26 17:24:40,336 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6387919 virtual) 2018-03-26 17:24:41,162 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6629596 virtual) 2018-03-26 17:24:44,895 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6859897 virtual) 2018-03-26 17:24:45,088 : INFO : 31 batches submitted to accumulate stats from 1984 documents (7078292 virtual) 2018-03-26 17:24:46,802 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7374715 virtual) 2018-03-26 17:24:48,529 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7576448 virtual) 2018-03-26 17:24:49,233 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7814680 virtual) 2018-03-26 17:24:51,455 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7991166 virtual) 2018-03-26 17:24:52,921 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8214683 virtual) 2018-03-26 17:24:53,173 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8413938 virtual) 2018-03-26 17:24:56,581 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8659392 virtual) 2018-03-26 17:24:57,117 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8888885 virtual) 2018-03-26 17:24:57,598 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9108513 virtual) 2018-03-26 17:25:00,031 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9298264 virtual) 2018-03-26 17:25:01,381 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9544627 virtual) 2018-03-26 17:25:01,591 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9860324 virtual) 2018-03-26 17:25:04,545 : INFO : 44 batches submitted to accumulate stats from 2816 documents (10069486 virtual) 2018-03-26 17:25:05,537 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10259283 virtual) 2018-03-26 17:25:05,732 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10618599 virtual) 2018-03-26 17:25:07,864 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10850507 virtual) 2018-03-26 17:25:10,069 : INFO : 48 batches submitted to accumulate stats from 3072 documents (11068746 virtual) 2018-03-26 17:25:11,127 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11260235 virtual) 2018-03-26 17:25:11,746 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11440306 virtual) 2018-03-26 17:25:13,483 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11862861 virtual) 2018-03-26 17:25:15,835 : INFO : 52 batches submitted to accumulate stats from 3328 documents (12180790 virtual) 2018-03-26 17:25:17,490 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12436827 virtual) 2018-03-26 17:25:17,635 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12694711 virtual) 2018-03-26 17:25:19,408 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12907287 virtual) 2018-03-26 17:25:20,654 : INFO : 56 batches submitted to accumulate stats from 3584 documents (13051609 virtual) 2018-03-26 17:25:27,349 : INFO : serializing accumulator to return to master... 2018-03-26 17:25:27,412 : INFO : accumulator serialized 2018-03-26 17:25:27,968 : INFO : serializing accumulator to return to master... 2018-03-26 17:25:27,975 : INFO : accumulator serialized 2018-03-26 17:25:28,376 : INFO : serializing accumulator to return to master... 2018-03-26 17:25:28,382 : INFO : accumulator serialized 2018-03-26 17:25:28,502 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:25:28,555 : INFO : accumulated word occurrence stats for 13051679 virtual documents 2018-03-26 17:25:28,740 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_15 2018-03-26 17:25:28,770 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_15.expElogbeta.npy with mmap=None 2018-03-26 17:25:28,826 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:25:28,827 : INFO : setting ignored attribute id2word to None 2018-03-26 17:25:28,828 : INFO : setting ignored attribute state to None 2018-03-26 17:25:28,828 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_15 2018-03-26 17:25:28,829 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_15.state 2018-03-26 17:25:28,934 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_15.state 2018-03-26 17:25:28,973 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:25:29,091 : INFO : 1 batches submitted to accumulate stats from 64 documents (231281 virtual) 2018-03-26 17:25:29,206 : INFO : 2 batches submitted to accumulate stats from 128 documents (559453 virtual) 2018-03-26 17:25:29,301 : INFO : 3 batches submitted to accumulate stats from 192 documents (769174 virtual) 2018-03-26 17:25:29,430 : INFO : 4 batches submitted to accumulate stats from 256 documents (990258 virtual) 2018-03-26 17:25:29,556 : INFO : 5 batches submitted to accumulate stats from 320 documents (1206756 virtual) 2018-03-26 17:25:29,671 : INFO : 6 batches submitted to accumulate stats from 384 documents (1420825 virtual) 2018-03-26 17:25:33,544 : INFO : 7 batches submitted to accumulate stats from 448 documents (1656938 virtual) 2018-03-26 17:25:34,019 : INFO : 8 batches submitted to accumulate stats from 512 documents (1850171 virtual) 2018-03-26 17:25:35,923 : INFO : 9 batches submitted to accumulate stats from 576 documents (2136550 virtual) 2018-03-26 17:25:38,140 : INFO : 10 batches submitted to accumulate stats from 640 documents (2339046 virtual) 2018-03-26 17:25:38,419 : INFO : 11 batches submitted to accumulate stats from 704 documents (2530358 virtual) 2018-03-26 17:25:40,282 : INFO : 12 batches submitted to accumulate stats from 768 documents (2711560 virtual) 2018-03-26 17:25:42,274 : INFO : 13 batches submitted to accumulate stats from 832 documents (2953166 virtual) 2018-03-26 17:25:42,759 : INFO : 14 batches submitted to accumulate stats from 896 documents (3167467 virtual) 2018-03-26 17:25:45,961 : INFO : 15 batches submitted to accumulate stats from 960 documents (3445606 virtual) 2018-03-26 17:25:46,202 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3666940 virtual) 2018-03-26 17:25:46,617 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3914084 virtual) 2018-03-26 17:25:49,584 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4156793 virtual) 2018-03-26 17:25:50,880 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4374402 virtual) 2018-03-26 17:25:51,008 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4598191 virtual) 2018-03-26 17:25:54,859 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4816189 virtual) 2018-03-26 17:25:55,368 : INFO : 22 batches submitted to accumulate stats from 1408 documents (5007308 virtual) 2018-03-26 17:25:55,872 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5242845 virtual) 2018-03-26 17:25:59,587 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5473175 virtual) 2018-03-26 17:25:59,746 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5678487 virtual) 2018-03-26 17:26:00,436 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5909047 virtual) 2018-03-26 17:26:03,371 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6144568 virtual) 2018-03-26 17:26:03,938 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6348383 virtual) 2018-03-26 17:26:05,011 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6576919 virtual) 2018-03-26 17:26:07,816 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6799947 virtual) 2018-03-26 17:26:08,090 : INFO : 31 batches submitted to accumulate stats from 1984 documents (7003398 virtual) 2018-03-26 17:26:09,558 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7229746 virtual) 2018-03-26 17:26:12,029 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7515611 virtual) 2018-03-26 17:26:12,536 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7734521 virtual) 2018-03-26 17:26:13,838 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7925119 virtual) 2018-03-26 17:26:16,240 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8167784 virtual) 2018-03-26 17:26:16,636 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8349050 virtual) 2018-03-26 17:26:18,257 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8603682 virtual) 2018-03-26 17:26:20,985 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8818932 virtual) 2018-03-26 17:26:21,873 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9052264 virtual) 2018-03-26 17:26:22,115 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9229637 virtual) 2018-03-26 17:26:25,623 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9461829 virtual) 2018-03-26 17:26:25,813 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9791136 virtual) 2018-03-26 17:26:26,989 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9991929 virtual) 2018-03-26 17:26:29,965 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10174425 virtual) 2018-03-26 17:26:30,218 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10491649 virtual) 2018-03-26 17:26:30,494 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10764539 virtual) 2018-03-26 17:26:34,403 : INFO : 48 batches submitted to accumulate stats from 3072 documents (11012060 virtual) 2018-03-26 17:26:34,631 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11188647 virtual) 2018-03-26 17:26:36,740 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11383859 virtual) 2018-03-26 17:26:37,928 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11736222 virtual) 2018-03-26 17:26:40,719 : INFO : 52 batches submitted to accumulate stats from 3328 documents (12107946 virtual) 2018-03-26 17:26:42,021 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12358484 virtual) 2018-03-26 17:26:42,831 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12610690 virtual) 2018-03-26 17:26:44,200 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12799716 virtual) 2018-03-26 17:26:45,954 : INFO : 56 batches submitted to accumulate stats from 3584 documents (13052258 virtual) 2018-03-26 17:26:54,077 : INFO : serializing accumulator to return to master... 2018-03-26 17:26:54,130 : INFO : accumulator serialized 2018-03-26 17:26:54,428 : INFO : serializing accumulator to return to master... 2018-03-26 17:26:54,434 : INFO : accumulator serialized 2018-03-26 17:26:55,777 : INFO : serializing accumulator to return to master... 2018-03-26 17:26:55,785 : INFO : accumulator serialized 2018-03-26 17:26:55,930 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:26:56,057 : INFO : accumulated word occurrence stats for 13052341 virtual documents 2018-03-26 17:26:56,287 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_20 2018-03-26 17:26:56,423 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_20.expElogbeta.npy with mmap=None 2018-03-26 17:26:56,489 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:26:56,489 : INFO : setting ignored attribute id2word to None 2018-03-26 17:26:56,490 : INFO : setting ignored attribute state to None 2018-03-26 17:26:56,491 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_20 2018-03-26 17:26:56,491 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_20.state 2018-03-26 17:26:56,685 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_20.state 2018-03-26 17:26:56,743 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:26:56,866 : INFO : 1 batches submitted to accumulate stats from 64 documents (230352 virtual) 2018-03-26 17:26:56,979 : INFO : 2 batches submitted to accumulate stats from 128 documents (559397 virtual) 2018-03-26 17:26:57,070 : INFO : 3 batches submitted to accumulate stats from 192 documents (764119 virtual) 2018-03-26 17:26:57,196 : INFO : 4 batches submitted to accumulate stats from 256 documents (984084 virtual) 2018-03-26 17:26:57,325 : INFO : 5 batches submitted to accumulate stats from 320 documents (1202061 virtual) 2018-03-26 17:26:57,446 : INFO : 6 batches submitted to accumulate stats from 384 documents (1409589 virtual) 2018-03-26 17:27:01,299 : INFO : 7 batches submitted to accumulate stats from 448 documents (1643855 virtual) 2018-03-26 17:27:01,783 : INFO : 8 batches submitted to accumulate stats from 512 documents (1841456 virtual) 2018-03-26 17:27:04,088 : INFO : 9 batches submitted to accumulate stats from 576 documents (2126565 virtual) 2018-03-26 17:27:05,695 : INFO : 10 batches submitted to accumulate stats from 640 documents (2333631 virtual) 2018-03-26 17:27:06,503 : INFO : 11 batches submitted to accumulate stats from 704 documents (2530202 virtual) 2018-03-26 17:27:08,412 : INFO : 12 batches submitted to accumulate stats from 768 documents (2708934 virtual) 2018-03-26 17:27:10,418 : INFO : 13 batches submitted to accumulate stats from 832 documents (2949095 virtual) 2018-03-26 17:27:10,642 : INFO : 14 batches submitted to accumulate stats from 896 documents (3166182 virtual) 2018-03-26 17:27:14,429 : INFO : 15 batches submitted to accumulate stats from 960 documents (3437861 virtual) 2018-03-26 17:27:14,734 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3655177 virtual) 2018-03-26 17:27:14,885 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3907894 virtual) 2018-03-26 17:27:18,093 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4152711 virtual) 2018-03-26 17:27:19,457 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4362842 virtual) 2018-03-26 17:27:19,946 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4594681 virtual) 2018-03-26 17:27:23,748 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4810487 virtual) 2018-03-26 17:27:24,171 : INFO : 22 batches submitted to accumulate stats from 1408 documents (5002194 virtual) 2018-03-26 17:27:25,496 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5235575 virtual) 2018-03-26 17:27:28,912 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5459548 virtual) 2018-03-26 17:27:29,274 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5666841 virtual) 2018-03-26 17:27:30,627 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5895843 virtual) 2018-03-26 17:27:33,144 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6121967 virtual) 2018-03-26 17:27:33,554 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6343148 virtual) 2018-03-26 17:27:35,460 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6569836 virtual) 2018-03-26 17:27:37,677 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6789309 virtual) 2018-03-26 17:27:37,948 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6994811 virtual) 2018-03-26 17:27:40,141 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7229433 virtual) 2018-03-26 17:27:42,405 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7500323 virtual) 2018-03-26 17:27:42,533 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7721342 virtual) 2018-03-26 17:27:44,579 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7919683 virtual) 2018-03-26 17:27:46,541 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8158778 virtual) 2018-03-26 17:27:47,169 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8345397 virtual) 2018-03-26 17:27:49,364 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8586213 virtual) 2018-03-26 17:27:51,892 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8807825 virtual) 2018-03-26 17:27:52,196 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9040425 virtual) 2018-03-26 17:27:53,435 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9218460 virtual) 2018-03-26 17:27:56,167 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9448725 virtual) 2018-03-26 17:27:56,848 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9775492 virtual) 2018-03-26 17:27:58,316 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9974446 virtual) 2018-03-26 17:28:00,946 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10169686 virtual) 2018-03-26 17:28:01,671 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10469976 virtual) 2018-03-26 17:28:02,049 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10740142 virtual) 2018-03-26 17:28:05,784 : INFO : 48 batches submitted to accumulate stats from 3072 documents (11001644 virtual) 2018-03-26 17:28:06,221 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11164024 virtual) 2018-03-26 17:28:08,494 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11359038 virtual) 2018-03-26 17:28:09,930 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11637933 virtual) 2018-03-26 17:28:12,573 : INFO : 52 batches submitted to accumulate stats from 3328 documents (12096713 virtual) 2018-03-26 17:28:13,967 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12332461 virtual) 2018-03-26 17:28:15,375 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12582544 virtual) 2018-03-26 17:28:15,934 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12774562 virtual) 2018-03-26 17:28:17,883 : INFO : 56 batches submitted to accumulate stats from 3584 documents (13027501 virtual) 2018-03-26 17:28:20,761 : INFO : 57 batches submitted to accumulate stats from 3648 documents (13052301 virtual) 2018-03-26 17:28:26,474 : INFO : serializing accumulator to return to master... 2018-03-26 17:28:26,420 : INFO : serializing accumulator to return to master... 2018-03-26 17:28:26,483 : INFO : accumulator serialized 2018-03-26 17:28:26,483 : INFO : accumulator serialized 2018-03-26 17:28:29,036 : INFO : serializing accumulator to return to master... 2018-03-26 17:28:29,042 : INFO : accumulator serialized 2018-03-26 17:28:29,232 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:28:29,378 : INFO : accumulated word occurrence stats for 13052389 virtual documents 2018-03-26 17:28:29,682 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_25 2018-03-26 17:28:29,709 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-26 17:28:29,795 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:28:29,796 : INFO : setting ignored attribute id2word to None 2018-03-26 17:28:29,796 : INFO : setting ignored attribute state to None 2018-03-26 17:28:29,797 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_25 2018-03-26 17:28:29,797 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_25.state 2018-03-26 17:28:29,956 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_25.state 2018-03-26 17:28:30,000 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:28:30,124 : INFO : 1 batches submitted to accumulate stats from 64 documents (230352 virtual) 2018-03-26 17:28:30,236 : INFO : 2 batches submitted to accumulate stats from 128 documents (559397 virtual) 2018-03-26 17:28:30,332 : INFO : 3 batches submitted to accumulate stats from 192 documents (764119 virtual) 2018-03-26 17:28:30,463 : INFO : 4 batches submitted to accumulate stats from 256 documents (984084 virtual) 2018-03-26 17:28:30,595 : INFO : 5 batches submitted to accumulate stats from 320 documents (1202061 virtual) 2018-03-26 17:28:30,718 : INFO : 6 batches submitted to accumulate stats from 384 documents (1409589 virtual) 2018-03-26 17:28:34,838 : INFO : 7 batches submitted to accumulate stats from 448 documents (1643855 virtual) 2018-03-26 17:28:35,241 : INFO : 8 batches submitted to accumulate stats from 512 documents (1841456 virtual) 2018-03-26 17:28:37,628 : INFO : 9 batches submitted to accumulate stats from 576 documents (2126565 virtual) 2018-03-26 17:28:39,614 : INFO : 10 batches submitted to accumulate stats from 640 documents (2333631 virtual) 2018-03-26 17:28:39,870 : INFO : 11 batches submitted to accumulate stats from 704 documents (2530202 virtual) 2018-03-26 17:28:42,066 : INFO : 12 batches submitted to accumulate stats from 768 documents (2708934 virtual) 2018-03-26 17:28:44,042 : INFO : 13 batches submitted to accumulate stats from 832 documents (2949095 virtual) 2018-03-26 17:28:44,522 : INFO : 14 batches submitted to accumulate stats from 896 documents (3161770 virtual) 2018-03-26 17:28:48,203 : INFO : 15 batches submitted to accumulate stats from 960 documents (3433595 virtual) 2018-03-26 17:28:48,528 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3653166 virtual) 2018-03-26 17:28:48,675 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3888639 virtual) 2018-03-26 17:28:51,933 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4145210 virtual) 2018-03-26 17:28:53,131 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4352524 virtual) 2018-03-26 17:28:53,692 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4581199 virtual) 2018-03-26 17:28:57,871 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4799816 virtual) 2018-03-26 17:28:57,990 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4995620 virtual) 2018-03-26 17:28:58,860 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5221976 virtual) 2018-03-26 17:29:02,526 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5448528 virtual) 2018-03-26 17:29:03,396 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5642426 virtual) 2018-03-26 17:29:03,874 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5885523 virtual) 2018-03-26 17:29:07,149 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6115460 virtual) 2018-03-26 17:29:07,659 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6337698 virtual) 2018-03-26 17:29:08,819 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6549463 virtual) 2018-03-26 17:29:12,020 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6759897 virtual) 2018-03-26 17:29:12,145 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6975998 virtual) 2018-03-26 17:29:13,901 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7213305 virtual) 2018-03-26 17:29:16,788 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7479801 virtual) 2018-03-26 17:29:17,095 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7713294 virtual) 2018-03-26 17:29:18,573 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7905043 virtual) 2018-03-26 17:29:21,428 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8136209 virtual) 2018-03-26 17:29:21,605 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8321935 virtual) 2018-03-26 17:29:23,709 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8578620 virtual) 2018-03-26 17:29:26,557 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8795567 virtual) 2018-03-26 17:29:27,214 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9024677 virtual) 2018-03-26 17:29:27,918 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9208981 virtual) 2018-03-26 17:29:31,396 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9427891 virtual) 2018-03-26 17:29:31,609 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9766159 virtual) 2018-03-26 17:29:33,346 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9952800 virtual) 2018-03-26 17:29:36,142 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10158712 virtual) 2018-03-26 17:29:36,562 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10427573 virtual) 2018-03-26 17:29:37,329 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10723068 virtual) 2018-03-26 17:29:40,868 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10945449 virtual) 2018-03-26 17:29:41,513 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11144238 virtual) 2018-03-26 17:29:43,904 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11348032 virtual) 2018-03-26 17:29:45,376 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11586244 virtual) 2018-03-26 17:29:47,320 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11943643 virtual) 2018-03-26 17:29:50,108 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12293656 virtual) 2018-03-26 17:29:50,397 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12543477 virtual) 2018-03-26 17:29:51,528 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12732815 virtual) 2018-03-26 17:29:54,493 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12999301 virtual) 2018-03-26 17:29:54,721 : INFO : 57 batches submitted to accumulate stats from 3648 documents (13052325 virtual) 2018-03-26 17:30:02,807 : INFO : serializing accumulator to return to master... 2018-03-26 17:30:02,868 : INFO : accumulator serialized 2018-03-26 17:30:03,178 : INFO : serializing accumulator to return to master... 2018-03-26 17:30:03,185 : INFO : accumulator serialized 2018-03-26 17:30:04,692 : INFO : serializing accumulator to return to master... 2018-03-26 17:30:04,699 : INFO : accumulator serialized 2018-03-26 17:30:04,936 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:30:05,132 : INFO : accumulated word occurrence stats for 13052417 virtual documents 2018-03-26 17:30:05,521 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_30 2018-03-26 17:30:05,567 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-26 17:30:05,780 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:30:05,780 : INFO : setting ignored attribute id2word to None 2018-03-26 17:30:05,781 : INFO : setting ignored attribute state to None 2018-03-26 17:30:05,781 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_30 2018-03-26 17:30:05,782 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_30.state 2018-03-26 17:30:06,079 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_30.state 2018-03-26 17:30:06,194 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:30:06,319 : INFO : 1 batches submitted to accumulate stats from 64 documents (230352 virtual) 2018-03-26 17:30:06,436 : INFO : 2 batches submitted to accumulate stats from 128 documents (559397 virtual) 2018-03-26 17:30:06,531 : INFO : 3 batches submitted to accumulate stats from 192 documents (764119 virtual) 2018-03-26 17:30:06,657 : INFO : 4 batches submitted to accumulate stats from 256 documents (984084 virtual) 2018-03-26 17:30:06,789 : INFO : 5 batches submitted to accumulate stats from 320 documents (1202061 virtual) 2018-03-26 17:30:06,919 : INFO : 6 batches submitted to accumulate stats from 384 documents (1404995 virtual) 2018-03-26 17:30:11,287 : INFO : 7 batches submitted to accumulate stats from 448 documents (1636823 virtual) 2018-03-26 17:30:11,674 : INFO : 8 batches submitted to accumulate stats from 512 documents (1841427 virtual) 2018-03-26 17:30:14,129 : INFO : 9 batches submitted to accumulate stats from 576 documents (2125656 virtual) 2018-03-26 17:30:16,362 : INFO : 10 batches submitted to accumulate stats from 640 documents (2332404 virtual) 2018-03-26 17:30:16,533 : INFO : 11 batches submitted to accumulate stats from 704 documents (2529581 virtual) 2018-03-26 17:30:18,722 : INFO : 12 batches submitted to accumulate stats from 768 documents (2703754 virtual) 2018-03-26 17:30:21,035 : INFO : 13 batches submitted to accumulate stats from 832 documents (2944401 virtual) 2018-03-26 17:30:21,286 : INFO : 14 batches submitted to accumulate stats from 896 documents (3161780 virtual) 2018-03-26 17:30:25,233 : INFO : 15 batches submitted to accumulate stats from 960 documents (3433605 virtual) 2018-03-26 17:30:25,677 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3655084 virtual) 2018-03-26 17:30:25,838 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3888646 virtual) 2018-03-26 17:30:29,036 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4145217 virtual) 2018-03-26 17:30:30,838 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4352531 virtual) 2018-03-26 17:30:31,165 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4581206 virtual) 2018-03-26 17:30:35,219 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4799823 virtual) 2018-03-26 17:30:35,860 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4995627 virtual) 2018-03-26 17:30:36,494 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5221983 virtual) 2018-03-26 17:30:40,687 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5448535 virtual) 2018-03-26 17:30:41,005 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5642433 virtual) 2018-03-26 17:30:42,021 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5885530 virtual) 2018-03-26 17:30:45,411 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6115467 virtual) 2018-03-26 17:30:45,543 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6337705 virtual) 2018-03-26 17:30:47,146 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6544919 virtual) 2018-03-26 17:30:49,831 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6759893 virtual) 2018-03-26 17:30:50,594 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6975994 virtual) 2018-03-26 17:30:52,612 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7209506 virtual) 2018-03-26 17:30:54,976 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7475553 virtual) 2018-03-26 17:30:55,588 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7709220 virtual) 2018-03-26 17:30:57,460 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7903239 virtual) 2018-03-26 17:30:59,823 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8136117 virtual) 2018-03-26 17:31:00,301 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8321838 virtual) 2018-03-26 17:31:02,631 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8567438 virtual) 2018-03-26 17:31:05,415 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8790276 virtual) 2018-03-26 17:31:05,777 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9009423 virtual) 2018-03-26 17:31:07,049 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9200375 virtual) 2018-03-26 17:31:10,021 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9427888 virtual) 2018-03-26 17:31:10,665 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9766156 virtual) 2018-03-26 17:31:12,630 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9952797 virtual) 2018-03-26 17:31:15,113 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10158709 virtual) 2018-03-26 17:31:15,439 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10427570 virtual) 2018-03-26 17:31:16,789 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10723065 virtual) 2018-03-26 17:31:20,249 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10949652 virtual) 2018-03-26 17:31:20,835 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11149261 virtual) 2018-03-26 17:31:22,991 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11349591 virtual) 2018-03-26 17:31:24,852 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11595569 virtual) 2018-03-26 17:31:26,946 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11943663 virtual) 2018-03-26 17:31:29,601 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12300259 virtual) 2018-03-26 17:31:29,945 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12552436 virtual) 2018-03-26 17:31:31,334 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12737406 virtual) 2018-03-26 17:31:33,973 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12999367 virtual) 2018-03-26 17:31:34,836 : INFO : 57 batches submitted to accumulate stats from 3648 documents (13052317 virtual) 2018-03-26 17:31:42,667 : INFO : serializing accumulator to return to master... 2018-03-26 17:31:42,722 : INFO : accumulator serialized 2018-03-26 17:31:42,981 : INFO : serializing accumulator to return to master... 2018-03-26 17:31:42,986 : INFO : accumulator serialized 2018-03-26 17:31:45,100 : INFO : serializing accumulator to return to master... 2018-03-26 17:31:45,105 : INFO : accumulator serialized 2018-03-26 17:31:45,391 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:31:45,640 : INFO : accumulated word occurrence stats for 13052412 virtual documents 2018-03-26 17:31:46,079 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_35 2018-03-26 17:31:46,108 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_35.expElogbeta.npy with mmap=None 2018-03-26 17:31:46,215 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:31:46,216 : INFO : setting ignored attribute id2word to None 2018-03-26 17:31:46,216 : INFO : setting ignored attribute state to None 2018-03-26 17:31:46,217 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_35 2018-03-26 17:31:46,217 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_35.state 2018-03-26 17:31:46,470 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_35.state 2018-03-26 17:31:46,537 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:31:46,664 : INFO : 1 batches submitted to accumulate stats from 64 documents (230352 virtual) 2018-03-26 17:31:46,784 : INFO : 2 batches submitted to accumulate stats from 128 documents (559397 virtual) 2018-03-26 17:31:46,886 : INFO : 3 batches submitted to accumulate stats from 192 documents (769182 virtual) 2018-03-26 17:31:47,008 : INFO : 4 batches submitted to accumulate stats from 256 documents (984088 virtual) 2018-03-26 17:31:47,147 : INFO : 5 batches submitted to accumulate stats from 320 documents (1206717 virtual) 2018-03-26 17:31:47,272 : INFO : 6 batches submitted to accumulate stats from 384 documents (1409603 virtual) 2018-03-26 17:31:51,696 : INFO : 7 batches submitted to accumulate stats from 448 documents (1643869 virtual) 2018-03-26 17:31:52,146 : INFO : 8 batches submitted to accumulate stats from 512 documents (1841470 virtual) 2018-03-26 17:31:54,758 : INFO : 9 batches submitted to accumulate stats from 576 documents (2126579 virtual) 2018-03-26 17:31:56,728 : INFO : 10 batches submitted to accumulate stats from 640 documents (2333645 virtual) 2018-03-26 17:31:57,405 : INFO : 11 batches submitted to accumulate stats from 704 documents (2530216 virtual) 2018-03-26 17:31:59,327 : INFO : 12 batches submitted to accumulate stats from 768 documents (2708948 virtual) 2018-03-26 17:32:02,016 : INFO : 13 batches submitted to accumulate stats from 832 documents (2949109 virtual) 2018-03-26 17:32:02,147 : INFO : 14 batches submitted to accumulate stats from 896 documents (3161784 virtual) 2018-03-26 17:32:06,111 : INFO : 15 batches submitted to accumulate stats from 960 documents (3433609 virtual) 2018-03-26 17:32:06,334 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3653180 virtual) 2018-03-26 17:32:06,931 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3888555 virtual) 2018-03-26 17:32:10,287 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4140500 virtual) 2018-03-26 17:32:11,766 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4348104 virtual) 2018-03-26 17:32:12,078 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4581142 virtual) 2018-03-26 17:32:16,438 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4794963 virtual) 2018-03-26 17:32:17,047 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4995635 virtual) 2018-03-26 17:32:17,399 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5214846 virtual) 2018-03-26 17:32:21,960 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5445551 virtual) 2018-03-26 17:32:22,452 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5635855 virtual) 2018-03-26 17:32:22,734 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5884577 virtual) 2018-03-26 17:32:26,897 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6111350 virtual) 2018-03-26 17:32:27,060 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6337664 virtual) 2018-03-26 17:32:27,873 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6541797 virtual) 2018-03-26 17:32:31,435 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6752412 virtual) 2018-03-26 17:32:32,334 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6969371 virtual) 2018-03-26 17:32:33,497 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7199112 virtual) 2018-03-26 17:32:36,711 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7467096 virtual) 2018-03-26 17:32:37,551 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7703850 virtual) 2018-03-26 17:32:38,288 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7884462 virtual) 2018-03-26 17:32:41,809 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8130174 virtual) 2018-03-26 17:32:42,634 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8303486 virtual) 2018-03-26 17:32:44,164 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8559218 virtual) 2018-03-26 17:32:48,747 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8778782 virtual) 2018-03-26 17:32:48,874 : INFO : 40 batches submitted to accumulate stats from 2560 documents (8998046 virtual) 2018-03-26 17:32:48,993 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9183713 virtual) 2018-03-26 17:32:52,891 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9416643 virtual) 2018-03-26 17:32:54,484 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9759781 virtual) 2018-03-26 17:32:54,888 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9945666 virtual) 2018-03-26 17:32:57,965 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10155851 virtual) 2018-03-26 17:32:59,089 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10408012 virtual) 2018-03-26 17:32:59,847 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10711789 virtual) 2018-03-26 17:33:03,365 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10924373 virtual) 2018-03-26 17:33:04,165 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11129054 virtual) 2018-03-26 17:33:07,108 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11339944 virtual) 2018-03-26 17:33:08,251 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11573381 virtual) 2018-03-26 17:33:09,957 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11928446 virtual) 2018-03-26 17:33:13,066 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12292558 virtual) 2018-03-26 17:33:14,096 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12525322 virtual) 2018-03-26 17:33:14,763 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12727513 virtual) 2018-03-26 17:33:17,974 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12974377 virtual) 2018-03-26 17:33:18,790 : INFO : 57 batches submitted to accumulate stats from 3648 documents (13052341 virtual) 2018-03-26 17:33:27,431 : INFO : serializing accumulator to return to master... 2018-03-26 17:33:27,486 : INFO : accumulator serialized 2018-03-26 17:33:27,767 : INFO : serializing accumulator to return to master... 2018-03-26 17:33:27,774 : INFO : accumulator serialized 2018-03-26 17:33:29,122 : INFO : serializing accumulator to return to master... 2018-03-26 17:33:29,127 : INFO : accumulator serialized 2018-03-26 17:33:29,456 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:33:29,781 : INFO : accumulated word occurrence stats for 13052433 virtual documents 2018-03-26 17:33:30,270 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_40 2018-03-26 17:33:30,305 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-26 17:33:30,427 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:33:30,428 : INFO : setting ignored attribute id2word to None 2018-03-26 17:33:30,429 : INFO : setting ignored attribute state to None 2018-03-26 17:33:30,429 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_40 2018-03-26 17:33:30,430 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_40.state 2018-03-26 17:33:30,672 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_40.state 2018-03-26 17:33:30,745 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:33:30,870 : INFO : 1 batches submitted to accumulate stats from 64 documents (230352 virtual) 2018-03-26 17:33:30,988 : INFO : 2 batches submitted to accumulate stats from 128 documents (559397 virtual) 2018-03-26 17:33:31,098 : INFO : 3 batches submitted to accumulate stats from 192 documents (769182 virtual) 2018-03-26 17:33:31,226 : INFO : 4 batches submitted to accumulate stats from 256 documents (984088 virtual) 2018-03-26 17:33:31,359 : INFO : 5 batches submitted to accumulate stats from 320 documents (1206717 virtual) 2018-03-26 17:33:31,481 : INFO : 6 batches submitted to accumulate stats from 384 documents (1409603 virtual) 2018-03-26 17:33:36,039 : INFO : 7 batches submitted to accumulate stats from 448 documents (1643869 virtual) 2018-03-26 17:33:36,525 : INFO : 8 batches submitted to accumulate stats from 512 documents (1841446 virtual) 2018-03-26 17:33:38,932 : INFO : 9 batches submitted to accumulate stats from 576 documents (2125675 virtual) 2018-03-26 17:33:41,286 : INFO : 10 batches submitted to accumulate stats from 640 documents (2332423 virtual) 2018-03-26 17:33:41,916 : INFO : 11 batches submitted to accumulate stats from 704 documents (2529600 virtual) 2018-03-26 17:33:43,726 : INFO : 12 batches submitted to accumulate stats from 768 documents (2703773 virtual) 2018-03-26 17:33:46,327 : INFO : 13 batches submitted to accumulate stats from 832 documents (2944420 virtual) 2018-03-26 17:33:46,701 : INFO : 14 batches submitted to accumulate stats from 896 documents (3158639 virtual) 2018-03-26 17:33:50,395 : INFO : 15 batches submitted to accumulate stats from 960 documents (3433578 virtual) 2018-03-26 17:33:51,319 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3653065 virtual) 2018-03-26 17:33:51,460 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3888572 virtual) 2018-03-26 17:33:54,385 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4140517 virtual) 2018-03-26 17:33:56,424 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4348121 virtual) 2018-03-26 17:33:57,095 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4581229 virtual) 2018-03-26 17:34:00,856 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4799846 virtual) 2018-03-26 17:34:01,504 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4996990 virtual) 2018-03-26 17:34:02,632 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5222132 virtual) 2018-03-26 17:34:06,450 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5451093 virtual) 2018-03-26 17:34:06,749 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5653400 virtual) 2018-03-26 17:34:08,272 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5885704 virtual) 2018-03-26 17:34:11,295 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6115578 virtual) 2018-03-26 17:34:11,427 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6337796 virtual) 2018-03-26 17:34:13,660 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6549491 virtual) 2018-03-26 17:34:16,287 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6766759 virtual) 2018-03-26 17:34:16,621 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6979355 virtual) 2018-03-26 17:34:19,075 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7213325 virtual) 2018-03-26 17:34:21,620 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7479826 virtual) 2018-03-26 17:34:21,845 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7713319 virtual) 2018-03-26 17:34:23,825 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7905068 virtual) 2018-03-26 17:34:26,543 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8136234 virtual) 2018-03-26 17:34:26,855 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8321960 virtual) 2018-03-26 17:34:29,301 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8578645 virtual) 2018-03-26 17:34:32,267 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8795592 virtual) 2018-03-26 17:34:32,573 : INFO : 40 batches submitted to accumulate stats from 2560 documents (9024702 virtual) 2018-03-26 17:34:33,599 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9209006 virtual) 2018-03-26 17:34:36,986 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9427916 virtual) 2018-03-26 17:34:37,660 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9766184 virtual) 2018-03-26 17:34:39,296 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9952825 virtual) 2018-03-26 17:34:42,041 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10158737 virtual) 2018-03-26 17:34:42,784 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10427598 virtual) 2018-03-26 17:34:43,598 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10723093 virtual) 2018-03-26 17:34:47,110 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10945474 virtual) 2018-03-26 17:34:47,726 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11144263 virtual) 2018-03-26 17:34:50,624 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11348001 virtual) 2018-03-26 17:34:51,894 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11586162 virtual) 2018-03-26 17:34:53,692 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11936645 virtual) 2018-03-26 17:34:57,078 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12293577 virtual) 2018-03-26 17:34:57,276 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12540152 virtual) 2018-03-26 17:34:58,131 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12732684 virtual) 2018-03-26 17:35:01,865 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12994901 virtual) 2018-03-26 17:35:01,999 : INFO : 57 batches submitted to accumulate stats from 3648 documents (13052346 virtual) 2018-03-26 17:35:10,380 : INFO : serializing accumulator to return to master... 2018-03-26 17:35:10,387 : INFO : accumulator serialized 2018-03-26 17:35:10,780 : INFO : serializing accumulator to return to master... 2018-03-26 17:35:10,786 : INFO : accumulator serialized 2018-03-26 17:35:12,495 : INFO : serializing accumulator to return to master... 2018-03-26 17:35:12,501 : INFO : accumulator serialized 2018-03-26 17:35:12,833 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:35:13,192 : INFO : accumulated word occurrence stats for 13052439 virtual documents 2018-03-26 17:35:13,743 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_50 2018-03-26 17:35:13,827 : INFO : loading expElogbeta from ../src/tmrest//ldamodel_nb_topics_50.expElogbeta.npy with mmap=None 2018-03-26 17:35:13,973 : INFO : setting ignored attribute dispatcher to None 2018-03-26 17:35:13,974 : INFO : setting ignored attribute id2word to None 2018-03-26 17:35:13,975 : INFO : setting ignored attribute state to None 2018-03-26 17:35:13,975 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_50 2018-03-26 17:35:13,976 : INFO : loading LdaModel object from ../src/tmrest//ldamodel_nb_topics_50.state 2018-03-26 17:35:14,254 : INFO : loaded ../src/tmrest//ldamodel_nb_topics_50.state 2018-03-26 17:35:14,333 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-26 17:35:14,460 : INFO : 1 batches submitted to accumulate stats from 64 documents (230352 virtual) 2018-03-26 17:35:14,578 : INFO : 2 batches submitted to accumulate stats from 128 documents (559397 virtual) 2018-03-26 17:35:14,680 : INFO : 3 batches submitted to accumulate stats from 192 documents (769182 virtual) 2018-03-26 17:35:14,811 : INFO : 4 batches submitted to accumulate stats from 256 documents (984088 virtual) 2018-03-26 17:35:14,946 : INFO : 5 batches submitted to accumulate stats from 320 documents (1206717 virtual) 2018-03-26 17:35:15,071 : INFO : 6 batches submitted to accumulate stats from 384 documents (1409603 virtual) 2018-03-26 17:35:19,792 : INFO : 7 batches submitted to accumulate stats from 448 documents (1643869 virtual) 2018-03-26 17:35:20,091 : INFO : 8 batches submitted to accumulate stats from 512 documents (1841446 virtual) 2018-03-26 17:35:22,809 : INFO : 9 batches submitted to accumulate stats from 576 documents (2125675 virtual) 2018-03-26 17:35:25,161 : INFO : 10 batches submitted to accumulate stats from 640 documents (2323488 virtual) 2018-03-26 17:35:25,732 : INFO : 11 batches submitted to accumulate stats from 704 documents (2529499 virtual) 2018-03-26 17:35:28,371 : INFO : 12 batches submitted to accumulate stats from 768 documents (2702828 virtual) 2018-03-26 17:35:30,684 : INFO : 13 batches submitted to accumulate stats from 832 documents (2944286 virtual) 2018-03-26 17:35:31,207 : INFO : 14 batches submitted to accumulate stats from 896 documents (3155962 virtual) 2018-03-26 17:35:35,488 : INFO : 15 batches submitted to accumulate stats from 960 documents (3426003 virtual) 2018-03-26 17:35:35,633 : INFO : 16 batches submitted to accumulate stats from 1024 documents (3652970 virtual) 2018-03-26 17:35:35,958 : INFO : 17 batches submitted to accumulate stats from 1088 documents (3888435 virtual) 2018-03-26 17:35:39,595 : INFO : 18 batches submitted to accumulate stats from 1152 documents (4131164 virtual) 2018-03-26 17:35:40,944 : INFO : 19 batches submitted to accumulate stats from 1216 documents (4346215 virtual) 2018-03-26 17:35:41,612 : INFO : 20 batches submitted to accumulate stats from 1280 documents (4569973 virtual) 2018-03-26 17:35:46,067 : INFO : 21 batches submitted to accumulate stats from 1344 documents (4789055 virtual) 2018-03-26 17:35:46,631 : INFO : 22 batches submitted to accumulate stats from 1408 documents (4986968 virtual) 2018-03-26 17:35:47,161 : INFO : 23 batches submitted to accumulate stats from 1472 documents (5201731 virtual) 2018-03-26 17:35:51,859 : INFO : 24 batches submitted to accumulate stats from 1536 documents (5431479 virtual) 2018-03-26 17:35:52,030 : INFO : 25 batches submitted to accumulate stats from 1600 documents (5620766 virtual) 2018-03-26 17:35:52,692 : INFO : 26 batches submitted to accumulate stats from 1664 documents (5878039 virtual) 2018-03-26 17:35:56,759 : INFO : 27 batches submitted to accumulate stats from 1728 documents (6111135 virtual) 2018-03-26 17:35:57,050 : INFO : 28 batches submitted to accumulate stats from 1792 documents (6324451 virtual) 2018-03-26 17:35:57,839 : INFO : 29 batches submitted to accumulate stats from 1856 documents (6537590 virtual) 2018-03-26 17:36:01,752 : INFO : 30 batches submitted to accumulate stats from 1920 documents (6723312 virtual) 2018-03-26 17:36:02,333 : INFO : 31 batches submitted to accumulate stats from 1984 documents (6958421 virtual) 2018-03-26 17:36:04,084 : INFO : 32 batches submitted to accumulate stats from 2048 documents (7179000 virtual) 2018-03-26 17:36:07,409 : INFO : 33 batches submitted to accumulate stats from 2112 documents (7456322 virtual) 2018-03-26 17:36:07,551 : INFO : 34 batches submitted to accumulate stats from 2176 documents (7679757 virtual) 2018-03-26 17:36:09,078 : INFO : 35 batches submitted to accumulate stats from 2240 documents (7873432 virtual) 2018-03-26 17:36:11,969 : INFO : 36 batches submitted to accumulate stats from 2304 documents (8113009 virtual) 2018-03-26 17:36:12,873 : INFO : 37 batches submitted to accumulate stats from 2368 documents (8275586 virtual) 2018-03-26 17:36:14,472 : INFO : 38 batches submitted to accumulate stats from 2432 documents (8536239 virtual) 2018-03-26 17:36:18,257 : INFO : 39 batches submitted to accumulate stats from 2496 documents (8747997 virtual) 2018-03-26 17:36:18,681 : INFO : 40 batches submitted to accumulate stats from 2560 documents (8969582 virtual) 2018-03-26 17:36:19,253 : INFO : 41 batches submitted to accumulate stats from 2624 documents (9167439 virtual) 2018-03-26 17:36:22,583 : INFO : 42 batches submitted to accumulate stats from 2688 documents (9387169 virtual) 2018-03-26 17:36:24,161 : INFO : 43 batches submitted to accumulate stats from 2752 documents (9724487 virtual) 2018-03-26 17:36:25,493 : INFO : 44 batches submitted to accumulate stats from 2816 documents (9919629 virtual) 2018-03-26 17:36:27,692 : INFO : 45 batches submitted to accumulate stats from 2880 documents (10136908 virtual) 2018-03-26 17:36:29,603 : INFO : 46 batches submitted to accumulate stats from 2944 documents (10376105 virtual) 2018-03-26 17:36:30,380 : INFO : 47 batches submitted to accumulate stats from 3008 documents (10692493 virtual) 2018-03-26 17:36:32,899 : INFO : 48 batches submitted to accumulate stats from 3072 documents (10914222 virtual) 2018-03-26 17:36:35,020 : INFO : 49 batches submitted to accumulate stats from 3136 documents (11109471 virtual) 2018-03-26 17:36:37,900 : INFO : 50 batches submitted to accumulate stats from 3200 documents (11328048 virtual) 2018-03-26 17:36:38,257 : INFO : 51 batches submitted to accumulate stats from 3264 documents (11565278 virtual) 2018-03-26 17:36:40,769 : INFO : 52 batches submitted to accumulate stats from 3328 documents (11907116 virtual) 2018-03-26 17:36:43,520 : INFO : 53 batches submitted to accumulate stats from 3392 documents (12255679 virtual) 2018-03-26 17:36:45,233 : INFO : 54 batches submitted to accumulate stats from 3456 documents (12493389 virtual) 2018-03-26 17:36:45,554 : INFO : 55 batches submitted to accumulate stats from 3520 documents (12720845 virtual) 2018-03-26 17:36:49,022 : INFO : 56 batches submitted to accumulate stats from 3584 documents (12940864 virtual) 2018-03-26 17:36:50,265 : INFO : 57 batches submitted to accumulate stats from 3648 documents (13052397 virtual) 2018-03-26 17:36:59,078 : INFO : serializing accumulator to return to master... 2018-03-26 17:36:59,086 : INFO : accumulator serialized 2018-03-26 17:36:59,857 : INFO : serializing accumulator to return to master... 2018-03-26 17:36:59,863 : INFO : accumulator serialized 2018-03-26 17:37:00,643 : INFO : serializing accumulator to return to master... 2018-03-26 17:37:00,649 : INFO : accumulator serialized 2018-03-26 17:37:01,109 : INFO : 3 accumulators retrieved from output queue 2018-03-26 17:37:01,635 : INFO : accumulated word occurrence stats for 13052498 virtual documents
Best number of topics¶
In [18]:
f = plt.figure(figsize=(8,8))
ax1 = f.add_subplot(211)
ax1 = tmrest_full_score.plot(x='num_topic',y='c_v',ax=ax1)
ax1 = tmrest_full_score.plot(x='num_topic', y='u_mass', ax=ax1, secondary_y=True)
ax2 = f.add_subplot(212)
ax2 = tmrest_full_score.plot(x='num_topic', y='c_npmi', ax=ax2)
ax2 = tmrest_full_score.plot(x='num_topic', y='c_uci',ax=ax2, secondary_y=True)
Top terms for LDA¶
In [19]:
best_num_topics = 20
tmrest_best_model = models.LdaModel.load(
'../src/tmrest/ldamodel_nb_topics_{}'.format(best_num_topics))
2018-03-26 18:08:47,373 : INFO : loading LdaModel object from ../src/tmrest/ldamodel_nb_topics_20 2018-03-26 18:08:47,442 : INFO : loading expElogbeta from ../src/tmrest/ldamodel_nb_topics_20.expElogbeta.npy with mmap=None 2018-03-26 18:08:47,541 : INFO : setting ignored attribute dispatcher to None 2018-03-26 18:08:47,542 : INFO : setting ignored attribute id2word to None 2018-03-26 18:08:47,542 : INFO : setting ignored attribute state to None 2018-03-26 18:08:47,543 : INFO : loaded ../src/tmrest/ldamodel_nb_topics_20 2018-03-26 18:08:47,543 : INFO : loading LdaModel object from ../src/tmrest/ldamodel_nb_topics_20.state 2018-03-26 18:08:47,671 : INFO : loaded ../src/tmrest/ldamodel_nb_topics_20.state
In [22]:
for i in range(best_num_topics):
print('TOPIC {}'.format(i))
eg = tmrest_best_model.get_topic_terms(i, topn=15)
for (a,b) in eg:
print('{}\t\t{:.5f}'.format(dict_restricted[a], b))
print('\n\n')
TOPIC 0 protocol 0.01487 network 0.01110 security 0.00707 message 0.00605 attack 0.00572 client 0.00503 key 0.00503 server 0.00500 packet 0.00452 privacy 0.00423 mobile 0.00413 measurement 0.00369 channel 0.00330 rate 0.00322 communication 0.00322 TOPIC 1 cid 0.00917 process 0.00871 stochastic 0.00805 rate 0.00776 space 0.00676 component 0.00667 function 0.00623 simulation 0.00587 probability 0.00570 reaction 0.00568 equation 0.00563 agent 0.00523 behaviour 0.00490 transition 0.00452 population 0.00440 TOPIC 2 word 0.01050 language 0.01007 translation 0.01004 al 0.00833 et 0.00801 sentence 0.00610 et_al 0.00580 computational 0.00547 machine 0.00529 linguistics 0.00524 feature 0.00480 corpus 0.00444 association 0.00437 text 0.00428 english 0.00393 TOPIC 3 speech 0.03890 voice 0.01553 synthesis 0.01133 tree 0.00982 noise 0.00774 speech_synthesis 0.00695 synthetic 0.00685 speaker 0.00636 spectral 0.00611 signal 0.00543 intelligibility 0.00472 natural 0.00455 frequency 0.00434 sentence 0.00423 hmm 0.00414 TOPIC 4 cid 0.25461 type 0.02024 let 0.00583 function 0.00512 language 0.00462 rule 0.00383 calculus 0.00311 proof 0.00307 name 0.00294 trace 0.00293 semantics 0.00292 programming 0.00282 return 0.00279 expression 0.00273 variable 0.00271 TOPIC 5 cid 0.02605 let 0.00985 lemma 0.00839 tree 0.00819 proof 0.00759 theorem 0.00701 automaton 0.00638 transition 0.00609 polynomial 0.00528 finite 0.00517 since 0.00513 game 0.00497 path 0.00475 thus 0.00469 node 0.00438 TOPIC 6 cid 0.02652 distribution 0.01577 algorithm 0.01184 learning 0.01007 parameter 0.00905 function 0.00896 sample 0.00873 probability 0.00807 log 0.00658 variable 0.00590 inference 0.00527 random 0.00449 gaussian 0.00410 probabilistic 0.00406 bayesian 0.00406 TOPIC 7 neuron 0.01448 network 0.00903 neural 0.00747 activity 0.00726 stimulus 0.00682 spike 0.00671 cell 0.00651 response 0.00632 input 0.00585 synaptic 0.00492 population 0.00473 al 0.00457 et 0.00457 rate 0.00424 et_al 0.00416 TOPIC 8 image 0.01911 object 0.01212 class 0.00844 feature 0.00784 training 0.00633 learning 0.00516 detection 0.00421 video 0.00412 dataset 0.00402 scene 0.00395 label 0.00347 al 0.00345 performance 0.00337 recognition 0.00333 segmentation 0.00332 TOPIC 9 al 0.01062 et 0.01020 gene 0.00998 cell 0.00968 et_al 0.00956 protein 0.00763 fig 0.00566 expression 0.00553 doi 0.00430 figure 0.00332 network 0.00295 mouse 0.00294 activity 0.00292 change 0.00291 site 0.00290 TOPIC 10 query 0.03531 graph 0.01681 node 0.01274 database 0.01045 algorithm 0.00894 pattern 0.00751 edge 0.00613 match 0.00508 instance 0.00462 view 0.00443 answer 0.00438 constraint 0.00436 attribute 0.00435 tuple 0.00434 tuples 0.00433 TOPIC 11 memory 0.01043 performance 0.00841 code 0.00833 core 0.00705 cache 0.00679 instruction 0.00558 program 0.00512 figure 0.00470 execution 0.00412 block 0.00408 high 0.00406 size 0.00379 benchmark 0.00377 implementation 0.00364 power 0.00348 TOPIC 12 motion 0.00698 al 0.00654 figure 0.00640 et 0.00640 fig 0.00627 doi 0.00574 et_al 0.00534 subject 0.00438 point 0.00417 map 0.00388 position 0.00388 control 0.00387 different 0.00365 cluster 0.00323 visual 0.00313 TOPIC 13 word 0.00958 sentence 0.00683 language 0.00675 object 0.00649 category 0.00640 effect 0.00639 participant 0.00566 processing 0.00451 context 0.00440 study 0.00436 experiment 0.00435 task 0.00389 verb 0.00378 et 0.00367 al 0.00366 TOPIC 14 cid 0.03098 quantum 0.00834 signature 0.00773 algebra 0.00756 theorem 0.00740 proof 0.00720 cq 0.00640 lemma 0.00630 let 0.00626 category 0.00548 complexity 0.00537 graph 0.00486 since 0.00474 then 0.00464 fo 0.00434 TOPIC 15 speech 0.01603 feature 0.01277 training 0.00873 network 0.00859 acoustic 0.00770 speaker 0.00700 neural 0.00586 language 0.00518 recognition 0.00514 layer 0.00484 proc 0.00458 ieee 0.00444 vol 0.00435 word 0.00432 deep 0.00395 TOPIC 16 sc 0.01012 process 0.00807 simulation 0.00675 event 0.00616 operation 0.00541 energy 0.00538 environment 0.00425 supply 0.00374 different 0.00372 modelling 0.00367 explanation 0.00361 service 0.00355 physical 0.00328 dynamic 0.00327 knowledge 0.00327 TOPIC 17 agent 0.01795 task 0.01151 action 0.01031 robot 0.00737 game 0.00591 planning 0.00571 plan 0.00455 cloud 0.00449 algorithm 0.00420 workflow 0.00418 environment 0.00378 domain 0.00378 interaction 0.00366 goal 0.00361 human 0.00348 TOPIC 18 rule 0.00905 language 0.00711 proof 0.00629 semantics 0.00621 logic 0.00593 program 0.00527 theory 0.00452 ontology 0.00430 relation 0.00402 class 0.00400 variable 0.00396 type 0.00395 formula 0.00363 function 0.00360 reasoning 0.00342 TOPIC 19 social 0.00573 study 0.00432 project 0.00430 web 0.00405 http 0.00405 patient 0.00377 group 0.00376 technology 0.00365 design 0.00310 computer 0.00305 people 0.00294 health 0.00293 de 0.00291 tool 0.00277 process 0.00261
Visualisation¶
In [23]:
vis_data = gensimvis.prepare(tmrest_best_model, corpus_tmrest, dict_restricted)
/Users/weiting/miniconda3/envs/infnet3/lib/python3.6/site-packages/pyLDAvis/_prepare.py:387: DeprecationWarning: .ix is deprecated. Please use .loc for label based indexing or .iloc for positional indexing See the documentation here: http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated topic_term_dists = topic_term_dists.ix[topic_order]
In [27]:
pyLDAvis.display(vis_data)
Out[27]:
In [26]:
# save in html for viewing
pyLDAvis.save_html(vis_data, open('../src/tmrest/res/ldavis.html', 'w'))
tmrest_meta (tm-6yr): Metadata only¶
In [54]:
# Constrain the dataset
df_combined_toks.drop(
df_combined_toks[(df_combined_toks.year < 2012)
| (df_combined_toks.year > 2017)].index,
inplace=True)
# Load corpora
dict_tmrest_meta = Dictionary.load(
os.path.join(DATA_DIR, 'corpora', 'dictionary.less.meta'))
df_metadata = df_combined_toks[['year', 'toks_metada']]
# Create a bow tagging for each publication:
df_metadata['bow'] = df_metadata['toks_metada'].apply(dict_tmrest_meta.doc2bow)
# Generate a corpus based on the tokens, which we will be using later
corpus_tmrest_meta = df_metadata.bow.tolist()
text_tmrest_meta = df_metadata.toks_metada.tolist()
tmrest_meta_score = pd.read_csv(
os.path.join('../src', 'tmrest_meta', 'scores.csv'))
2018-03-28 20:23:51,086 : INFO : loading Dictionary object from ../../data/data_schoolofinf/corpora/dictionary.less.meta 2018-03-28 20:23:51,143 : INFO : loaded ../../data/data_schoolofinf/corpora/dictionary.less.meta
Evaluation of model¶
In [23]:
tmrest_meta_score['c_v'] = tmrest_meta_score.apply(lambda row: get_score(row.num_topic, dict_tmrest_meta, texts=text_tmrest_meta, corpus=corpus_tmrest_meta, model_dir='../src/tmrest_meta/',
coherence='c_v'),
axis=1)
2018-03-28 00:23:42,197 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_10 2018-03-28 00:23:42,220 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-28 00:23:42,239 : INFO : setting ignored attribute id2word to None 2018-03-28 00:23:42,239 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:23:42,240 : INFO : setting ignored attribute state to None 2018-03-28 00:23:42,241 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_10 2018-03-28 00:23:42,241 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_10.state 2018-03-28 00:23:42,265 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_10.state 2018-03-28 00:23:42,275 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:23:43,846 : INFO : 43 batches submitted to accumulate stats from 2752 documents (-37317 virtual) 2018-03-28 00:23:44,445 : INFO : serializing accumulator to return to master... 2018-03-28 00:23:44,434 : INFO : serializing accumulator to return to master... 2018-03-28 00:23:44,439 : INFO : serializing accumulator to return to master... 2018-03-28 00:23:44,507 : INFO : accumulator serialized 2018-03-28 00:23:44,510 : INFO : accumulator serialized 2018-03-28 00:23:44,506 : INFO : accumulator serialized 2018-03-28 00:23:44,576 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:23:44,628 : INFO : accumulated word occurrence stats for 48376 virtual documents 2018-03-28 00:23:44,971 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_15 2018-03-28 00:23:44,983 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_15.expElogbeta.npy with mmap=None 2018-03-28 00:23:44,993 : INFO : setting ignored attribute id2word to None 2018-03-28 00:23:44,994 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:23:44,995 : INFO : setting ignored attribute state to None 2018-03-28 00:23:44,996 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_15 2018-03-28 00:23:44,997 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_15.state 2018-03-28 00:23:45,021 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_15.state 2018-03-28 00:23:45,027 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:23:47,405 : INFO : serializing accumulator to return to master... 2018-03-28 00:23:47,421 : INFO : serializing accumulator to return to master... 2018-03-28 00:23:47,410 : INFO : accumulator serialized 2018-03-28 00:23:47,428 : INFO : accumulator serialized 2018-03-28 00:23:47,485 : INFO : serializing accumulator to return to master... 2018-03-28 00:23:47,491 : INFO : accumulator serialized 2018-03-28 00:23:47,539 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:23:47,617 : INFO : accumulated word occurrence stats for 48382 virtual documents 2018-03-28 00:23:48,080 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_20 2018-03-28 00:23:48,086 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_20.expElogbeta.npy with mmap=None 2018-03-28 00:23:48,096 : INFO : setting ignored attribute id2word to None 2018-03-28 00:23:48,097 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:23:48,097 : INFO : setting ignored attribute state to None 2018-03-28 00:23:48,098 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_20 2018-03-28 00:23:48,099 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_20.state 2018-03-28 00:23:48,118 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_20.state 2018-03-28 00:23:48,123 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:23:51,290 : INFO : serializing accumulator to return to master... 2018-03-28 00:23:51,297 : INFO : accumulator serialized 2018-03-28 00:23:51,406 : INFO : serializing accumulator to return to master... 2018-03-28 00:23:51,412 : INFO : serializing accumulator to return to master... 2018-03-28 00:23:51,412 : INFO : accumulator serialized 2018-03-28 00:23:51,418 : INFO : accumulator serialized 2018-03-28 00:23:51,508 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:23:51,632 : INFO : accumulated word occurrence stats for 48398 virtual documents 2018-03-28 00:23:52,309 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_25 2018-03-28 00:23:52,320 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-28 00:23:52,340 : INFO : setting ignored attribute id2word to None 2018-03-28 00:23:52,341 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:23:52,342 : INFO : setting ignored attribute state to None 2018-03-28 00:23:52,342 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_25 2018-03-28 00:23:52,343 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_25.state 2018-03-28 00:23:52,371 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_25.state 2018-03-28 00:23:52,377 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:23:56,269 : INFO : serializing accumulator to return to master... 2018-03-28 00:23:56,294 : INFO : serializing accumulator to return to master... 2018-03-28 00:23:56,275 : INFO : accumulator serialized 2018-03-28 00:23:56,381 : INFO : serializing accumulator to return to master... 2018-03-28 00:23:56,300 : INFO : accumulator serialized 2018-03-28 00:23:56,389 : INFO : accumulator serialized 2018-03-28 00:23:56,495 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:23:56,650 : INFO : accumulated word occurrence stats for 48403 virtual documents 2018-03-28 00:23:57,517 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_30 2018-03-28 00:23:57,558 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-28 00:23:57,569 : INFO : setting ignored attribute id2word to None 2018-03-28 00:23:57,569 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:23:57,570 : INFO : setting ignored attribute state to None 2018-03-28 00:23:57,570 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_30 2018-03-28 00:23:57,571 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_30.state 2018-03-28 00:23:57,598 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_30.state 2018-03-28 00:23:57,605 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:24:01,934 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:01,958 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:01,939 : INFO : accumulator serialized 2018-03-28 00:24:01,965 : INFO : accumulator serialized 2018-03-28 00:24:02,122 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:02,128 : INFO : accumulator serialized 2018-03-28 00:24:02,242 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:24:02,453 : INFO : accumulated word occurrence stats for 48403 virtual documents 2018-03-28 00:24:03,491 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_35 2018-03-28 00:24:03,531 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_35.expElogbeta.npy with mmap=None 2018-03-28 00:24:03,553 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:03,554 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:03,554 : INFO : setting ignored attribute state to None 2018-03-28 00:24:03,555 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_35 2018-03-28 00:24:03,556 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_35.state 2018-03-28 00:24:03,583 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_35.state 2018-03-28 00:24:03,591 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:24:08,387 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:08,410 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:08,453 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:08,394 : INFO : accumulator serialized 2018-03-28 00:24:08,417 : INFO : accumulator serialized 2018-03-28 00:24:08,459 : INFO : accumulator serialized 2018-03-28 00:24:08,654 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:24:08,896 : INFO : accumulated word occurrence stats for 48480 virtual documents 2018-03-28 00:24:10,105 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_40 2018-03-28 00:24:10,150 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-28 00:24:10,167 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:10,168 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:10,168 : INFO : setting ignored attribute state to None 2018-03-28 00:24:10,169 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_40 2018-03-28 00:24:10,169 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_40.state 2018-03-28 00:24:10,208 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_40.state 2018-03-28 00:24:10,218 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:24:15,907 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:15,988 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:16,010 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:15,913 : INFO : accumulator serialized 2018-03-28 00:24:15,995 : INFO : accumulator serialized 2018-03-28 00:24:16,017 : INFO : accumulator serialized 2018-03-28 00:24:16,255 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:24:16,568 : INFO : accumulated word occurrence stats for 48487 virtual documents 2018-03-28 00:24:18,005 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_50 2018-03-28 00:24:18,062 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_50.expElogbeta.npy with mmap=None 2018-03-28 00:24:18,155 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:18,156 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:18,156 : INFO : setting ignored attribute state to None 2018-03-28 00:24:18,157 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_50 2018-03-28 00:24:18,157 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_50.state 2018-03-28 00:24:18,196 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_50.state 2018-03-28 00:24:18,207 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:24:25,515 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:25,541 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:25,553 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:25,522 : INFO : accumulator serialized 2018-03-28 00:24:25,560 : INFO : accumulator serialized 2018-03-28 00:24:25,547 : INFO : accumulator serialized 2018-03-28 00:24:25,944 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:24:26,382 : INFO : accumulated word occurrence stats for 48486 virtual documents
In [24]:
tmrest_meta_score['u_mass'] = tmrest_meta_score.apply(lambda row: get_score(row.num_topic, dict_tmrest_meta, texts=text_tmrest_meta, corpus=corpus_tmrest_meta, model_dir='../src/tmrest_meta/',
coherence='u_mass'),
axis=1)
2018-03-28 00:24:28,652 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_10 2018-03-28 00:24:28,654 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-28 00:24:28,655 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:28,656 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:28,656 : INFO : setting ignored attribute state to None 2018-03-28 00:24:28,657 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_10 2018-03-28 00:24:28,657 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_10.state 2018-03-28 00:24:28,662 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_10.state 2018-03-28 00:24:28,687 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-28 00:24:28,707 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-28 00:24:28,727 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-28 00:24:28,766 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_15 2018-03-28 00:24:28,768 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_15.expElogbeta.npy with mmap=None 2018-03-28 00:24:28,769 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:28,770 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:28,770 : INFO : setting ignored attribute state to None 2018-03-28 00:24:28,771 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_15 2018-03-28 00:24:28,771 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_15.state 2018-03-28 00:24:28,777 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_15.state 2018-03-28 00:24:28,805 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-28 00:24:28,826 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-28 00:24:28,847 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-28 00:24:28,896 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_20 2018-03-28 00:24:28,897 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_20.expElogbeta.npy with mmap=None 2018-03-28 00:24:28,899 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:28,899 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:28,900 : INFO : setting ignored attribute state to None 2018-03-28 00:24:28,900 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_20 2018-03-28 00:24:28,901 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_20.state 2018-03-28 00:24:28,908 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_20.state 2018-03-28 00:24:28,938 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-28 00:24:28,961 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-28 00:24:28,986 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-28 00:24:29,042 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_25 2018-03-28 00:24:29,043 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-28 00:24:29,045 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:29,045 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:29,046 : INFO : setting ignored attribute state to None 2018-03-28 00:24:29,047 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_25 2018-03-28 00:24:29,047 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_25.state 2018-03-28 00:24:29,055 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_25.state 2018-03-28 00:24:29,087 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-28 00:24:29,112 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-28 00:24:29,137 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-28 00:24:29,199 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_30 2018-03-28 00:24:29,200 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-28 00:24:29,202 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:29,203 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:29,203 : INFO : setting ignored attribute state to None 2018-03-28 00:24:29,204 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_30 2018-03-28 00:24:29,205 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_30.state 2018-03-28 00:24:29,214 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_30.state 2018-03-28 00:24:29,247 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-28 00:24:29,273 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-28 00:24:29,298 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-28 00:24:29,369 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_35 2018-03-28 00:24:29,370 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_35.expElogbeta.npy with mmap=None 2018-03-28 00:24:29,372 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:29,373 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:29,373 : INFO : setting ignored attribute state to None 2018-03-28 00:24:29,374 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_35 2018-03-28 00:24:29,374 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_35.state 2018-03-28 00:24:29,384 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_35.state 2018-03-28 00:24:29,421 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-28 00:24:29,448 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-28 00:24:29,475 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-28 00:24:29,552 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_40 2018-03-28 00:24:29,553 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-28 00:24:29,556 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:29,557 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:29,557 : INFO : setting ignored attribute state to None 2018-03-28 00:24:29,558 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_40 2018-03-28 00:24:29,558 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_40.state 2018-03-28 00:24:29,569 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_40.state 2018-03-28 00:24:29,607 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-28 00:24:29,634 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-28 00:24:29,661 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-03-28 00:24:29,748 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_50 2018-03-28 00:24:29,749 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_50.expElogbeta.npy with mmap=None 2018-03-28 00:24:29,751 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:29,752 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:29,752 : INFO : setting ignored attribute state to None 2018-03-28 00:24:29,753 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_50 2018-03-28 00:24:29,753 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_50.state 2018-03-28 00:24:29,769 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_50.state 2018-03-28 00:24:29,809 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-03-28 00:24:29,837 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-03-28 00:24:29,866 : INFO : CorpusAccumulator accumulated stats from 3000 documents
In [25]:
tmrest_meta_score['c_npmi'] = tmrest_meta_score.apply(lambda row: get_score(row.num_topic, dict_tmrest_meta, texts=text_tmrest_meta, corpus=corpus_tmrest_meta, model_dir='../src/tmrest_meta/',
coherence='c_npmi'),
axis=1)
2018-03-28 00:24:29,966 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_10 2018-03-28 00:24:29,968 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-28 00:24:29,970 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:29,971 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:29,972 : INFO : setting ignored attribute state to None 2018-03-28 00:24:29,972 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_10 2018-03-28 00:24:29,973 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_10.state 2018-03-28 00:24:29,978 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_10.state 2018-03-28 00:24:29,985 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:24:30,025 : INFO : 1 batches submitted to accumulate stats from 64 documents (5664 virtual) 2018-03-28 00:24:30,033 : INFO : 2 batches submitted to accumulate stats from 128 documents (10490 virtual) 2018-03-28 00:24:30,040 : INFO : 3 batches submitted to accumulate stats from 192 documents (15843 virtual) 2018-03-28 00:24:30,047 : INFO : 4 batches submitted to accumulate stats from 256 documents (21113 virtual) 2018-03-28 00:24:30,055 : INFO : 5 batches submitted to accumulate stats from 320 documents (26825 virtual) 2018-03-28 00:24:30,061 : INFO : 6 batches submitted to accumulate stats from 384 documents (32204 virtual) 2018-03-28 00:24:30,189 : INFO : 7 batches submitted to accumulate stats from 448 documents (37741 virtual) 2018-03-28 00:24:30,205 : INFO : 8 batches submitted to accumulate stats from 512 documents (42827 virtual) 2018-03-28 00:24:30,212 : INFO : 9 batches submitted to accumulate stats from 576 documents (48357 virtual) 2018-03-28 00:24:30,352 : INFO : 10 batches submitted to accumulate stats from 640 documents (53652 virtual) 2018-03-28 00:24:30,358 : INFO : 11 batches submitted to accumulate stats from 704 documents (59783 virtual) 2018-03-28 00:24:30,372 : INFO : 12 batches submitted to accumulate stats from 768 documents (64830 virtual) 2018-03-28 00:24:30,505 : INFO : 13 batches submitted to accumulate stats from 832 documents (70885 virtual) 2018-03-28 00:24:30,512 : INFO : 14 batches submitted to accumulate stats from 896 documents (77030 virtual) 2018-03-28 00:24:30,523 : INFO : 15 batches submitted to accumulate stats from 960 documents (82155 virtual) 2018-03-28 00:24:30,652 : INFO : 16 batches submitted to accumulate stats from 1024 documents (87812 virtual) 2018-03-28 00:24:30,673 : INFO : 17 batches submitted to accumulate stats from 1088 documents (93804 virtual) 2018-03-28 00:24:30,685 : INFO : 18 batches submitted to accumulate stats from 1152 documents (99493 virtual) 2018-03-28 00:24:30,823 : INFO : 19 batches submitted to accumulate stats from 1216 documents (104728 virtual) 2018-03-28 00:24:30,829 : INFO : 20 batches submitted to accumulate stats from 1280 documents (109987 virtual) 2018-03-28 00:24:30,851 : INFO : 21 batches submitted to accumulate stats from 1344 documents (115191 virtual) 2018-03-28 00:24:30,971 : INFO : 22 batches submitted to accumulate stats from 1408 documents (120376 virtual) 2018-03-28 00:24:30,984 : INFO : 23 batches submitted to accumulate stats from 1472 documents (126058 virtual) 2018-03-28 00:24:31,031 : INFO : 24 batches submitted to accumulate stats from 1536 documents (131892 virtual) 2018-03-28 00:24:31,128 : INFO : 25 batches submitted to accumulate stats from 1600 documents (137760 virtual) 2018-03-28 00:24:31,135 : INFO : 26 batches submitted to accumulate stats from 1664 documents (143272 virtual) 2018-03-28 00:24:31,178 : INFO : 27 batches submitted to accumulate stats from 1728 documents (148240 virtual) 2018-03-28 00:24:31,273 : INFO : 28 batches submitted to accumulate stats from 1792 documents (153219 virtual) 2018-03-28 00:24:31,279 : INFO : 29 batches submitted to accumulate stats from 1856 documents (158552 virtual) 2018-03-28 00:24:31,370 : INFO : 30 batches submitted to accumulate stats from 1920 documents (163941 virtual) 2018-03-28 00:24:31,422 : INFO : 31 batches submitted to accumulate stats from 1984 documents (169520 virtual) 2018-03-28 00:24:31,434 : INFO : 32 batches submitted to accumulate stats from 2048 documents (175298 virtual) 2018-03-28 00:24:31,524 : INFO : 33 batches submitted to accumulate stats from 2112 documents (181535 virtual) 2018-03-28 00:24:31,555 : INFO : 34 batches submitted to accumulate stats from 2176 documents (186237 virtual) 2018-03-28 00:24:31,607 : INFO : 35 batches submitted to accumulate stats from 2240 documents (191702 virtual) 2018-03-28 00:24:31,675 : INFO : 36 batches submitted to accumulate stats from 2304 documents (197243 virtual) 2018-03-28 00:24:31,715 : INFO : 37 batches submitted to accumulate stats from 2368 documents (203024 virtual) 2018-03-28 00:24:31,759 : INFO : 38 batches submitted to accumulate stats from 2432 documents (208987 virtual) 2018-03-28 00:24:31,849 : INFO : 39 batches submitted to accumulate stats from 2496 documents (215014 virtual) 2018-03-28 00:24:31,869 : INFO : 40 batches submitted to accumulate stats from 2560 documents (220325 virtual) 2018-03-28 00:24:31,906 : INFO : 41 batches submitted to accumulate stats from 2624 documents (225941 virtual) 2018-03-28 00:24:31,991 : INFO : 42 batches submitted to accumulate stats from 2688 documents (231434 virtual) 2018-03-28 00:24:32,031 : INFO : 43 batches submitted to accumulate stats from 2752 documents (237883 virtual) 2018-03-28 00:24:32,083 : INFO : 44 batches submitted to accumulate stats from 2816 documents (243186 virtual) 2018-03-28 00:24:32,154 : INFO : 45 batches submitted to accumulate stats from 2880 documents (248092 virtual) 2018-03-28 00:24:32,178 : INFO : 46 batches submitted to accumulate stats from 2944 documents (253637 virtual) 2018-03-28 00:24:32,251 : INFO : 47 batches submitted to accumulate stats from 3008 documents (259111 virtual) 2018-03-28 00:24:32,311 : INFO : 48 batches submitted to accumulate stats from 3072 documents (264929 virtual) 2018-03-28 00:24:32,361 : INFO : 49 batches submitted to accumulate stats from 3136 documents (270342 virtual) 2018-03-28 00:24:32,427 : INFO : 50 batches submitted to accumulate stats from 3200 documents (275370 virtual) 2018-03-28 00:24:32,446 : INFO : 51 batches submitted to accumulate stats from 3264 documents (281280 virtual) 2018-03-28 00:24:32,536 : INFO : 52 batches submitted to accumulate stats from 3328 documents (286979 virtual) 2018-03-28 00:24:32,588 : INFO : 53 batches submitted to accumulate stats from 3392 documents (292361 virtual) 2018-03-28 00:24:32,624 : INFO : 54 batches submitted to accumulate stats from 3456 documents (298359 virtual) 2018-03-28 00:24:32,701 : INFO : 55 batches submitted to accumulate stats from 3520 documents (304492 virtual) 2018-03-28 00:24:32,742 : INFO : 56 batches submitted to accumulate stats from 3584 documents (308327 virtual) 2018-03-28 00:24:32,964 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:32,970 : INFO : accumulator serialized 2018-03-28 00:24:33,018 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:33,024 : INFO : accumulator serialized 2018-03-28 00:24:33,037 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:33,044 : INFO : accumulator serialized 2018-03-28 00:24:33,071 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:24:33,105 : INFO : accumulated word occurrence stats for 308484 virtual documents 2018-03-28 00:24:33,403 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_15 2018-03-28 00:24:33,405 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_15.expElogbeta.npy with mmap=None 2018-03-28 00:24:33,408 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:33,409 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:33,409 : INFO : setting ignored attribute state to None 2018-03-28 00:24:33,411 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_15 2018-03-28 00:24:33,411 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_15.state 2018-03-28 00:24:33,419 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_15.state 2018-03-28 00:24:33,431 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:24:33,467 : INFO : 1 batches submitted to accumulate stats from 64 documents (5664 virtual) 2018-03-28 00:24:33,476 : INFO : 2 batches submitted to accumulate stats from 128 documents (10533 virtual) 2018-03-28 00:24:33,483 : INFO : 3 batches submitted to accumulate stats from 192 documents (15989 virtual) 2018-03-28 00:24:33,491 : INFO : 4 batches submitted to accumulate stats from 256 documents (21250 virtual) 2018-03-28 00:24:33,499 : INFO : 5 batches submitted to accumulate stats from 320 documents (26908 virtual) 2018-03-28 00:24:33,507 : INFO : 6 batches submitted to accumulate stats from 384 documents (32308 virtual) 2018-03-28 00:24:33,622 : INFO : 7 batches submitted to accumulate stats from 448 documents (37734 virtual) 2018-03-28 00:24:33,660 : INFO : 8 batches submitted to accumulate stats from 512 documents (42666 virtual) 2018-03-28 00:24:33,700 : INFO : 9 batches submitted to accumulate stats from 576 documents (48261 virtual) 2018-03-28 00:24:33,801 : INFO : 10 batches submitted to accumulate stats from 640 documents (53570 virtual) 2018-03-28 00:24:33,833 : INFO : 11 batches submitted to accumulate stats from 704 documents (59720 virtual) 2018-03-28 00:24:33,860 : INFO : 12 batches submitted to accumulate stats from 768 documents (64712 virtual) 2018-03-28 00:24:33,977 : INFO : 13 batches submitted to accumulate stats from 832 documents (70811 virtual) 2018-03-28 00:24:33,994 : INFO : 14 batches submitted to accumulate stats from 896 documents (76931 virtual) 2018-03-28 00:24:34,029 : INFO : 15 batches submitted to accumulate stats from 960 documents (81830 virtual) 2018-03-28 00:24:34,138 : INFO : 16 batches submitted to accumulate stats from 1024 documents (87697 virtual) 2018-03-28 00:24:34,173 : INFO : 17 batches submitted to accumulate stats from 1088 documents (93729 virtual) 2018-03-28 00:24:34,201 : INFO : 18 batches submitted to accumulate stats from 1152 documents (99399 virtual) 2018-03-28 00:24:34,324 : INFO : 19 batches submitted to accumulate stats from 1216 documents (104563 virtual) 2018-03-28 00:24:34,360 : INFO : 20 batches submitted to accumulate stats from 1280 documents (109840 virtual) 2018-03-28 00:24:34,366 : INFO : 21 batches submitted to accumulate stats from 1344 documents (115021 virtual) 2018-03-28 00:24:34,475 : INFO : 22 batches submitted to accumulate stats from 1408 documents (119983 virtual) 2018-03-28 00:24:34,543 : INFO : 23 batches submitted to accumulate stats from 1472 documents (125885 virtual) 2018-03-28 00:24:34,554 : INFO : 24 batches submitted to accumulate stats from 1536 documents (131573 virtual) 2018-03-28 00:24:34,634 : INFO : 25 batches submitted to accumulate stats from 1600 documents (137596 virtual) 2018-03-28 00:24:34,691 : INFO : 26 batches submitted to accumulate stats from 1664 documents (142940 virtual) 2018-03-28 00:24:34,723 : INFO : 27 batches submitted to accumulate stats from 1728 documents (148087 virtual) 2018-03-28 00:24:34,815 : INFO : 28 batches submitted to accumulate stats from 1792 documents (152880 virtual) 2018-03-28 00:24:34,882 : INFO : 29 batches submitted to accumulate stats from 1856 documents (158259 virtual) 2018-03-28 00:24:34,925 : INFO : 30 batches submitted to accumulate stats from 1920 documents (163692 virtual) 2018-03-28 00:24:35,005 : INFO : 31 batches submitted to accumulate stats from 1984 documents (169283 virtual) 2018-03-28 00:24:35,049 : INFO : 32 batches submitted to accumulate stats from 2048 documents (175062 virtual) 2018-03-28 00:24:35,089 : INFO : 33 batches submitted to accumulate stats from 2112 documents (181235 virtual) 2018-03-28 00:24:35,152 : INFO : 34 batches submitted to accumulate stats from 2176 documents (185777 virtual) 2018-03-28 00:24:35,202 : INFO : 35 batches submitted to accumulate stats from 2240 documents (191334 virtual) 2018-03-28 00:24:35,268 : INFO : 36 batches submitted to accumulate stats from 2304 documents (197142 virtual) 2018-03-28 00:24:35,305 : INFO : 37 batches submitted to accumulate stats from 2368 documents (202665 virtual) 2018-03-28 00:24:35,405 : INFO : 38 batches submitted to accumulate stats from 2432 documents (208788 virtual) 2018-03-28 00:24:35,447 : INFO : 39 batches submitted to accumulate stats from 2496 documents (214934 virtual) 2018-03-28 00:24:35,481 : INFO : 40 batches submitted to accumulate stats from 2560 documents (219927 virtual) 2018-03-28 00:24:35,557 : INFO : 41 batches submitted to accumulate stats from 2624 documents (225532 virtual) 2018-03-28 00:24:35,618 : INFO : 42 batches submitted to accumulate stats from 2688 documents (231175 virtual) 2018-03-28 00:24:35,662 : INFO : 43 batches submitted to accumulate stats from 2752 documents (237272 virtual) 2018-03-28 00:24:35,730 : INFO : 44 batches submitted to accumulate stats from 2816 documents (242807 virtual) 2018-03-28 00:24:35,807 : INFO : 45 batches submitted to accumulate stats from 2880 documents (247586 virtual) 2018-03-28 00:24:35,832 : INFO : 46 batches submitted to accumulate stats from 2944 documents (253219 virtual) 2018-03-28 00:24:35,895 : INFO : 47 batches submitted to accumulate stats from 3008 documents (258976 virtual) 2018-03-28 00:24:35,971 : INFO : 48 batches submitted to accumulate stats from 3072 documents (264528 virtual) 2018-03-28 00:24:36,014 : INFO : 49 batches submitted to accumulate stats from 3136 documents (270009 virtual) 2018-03-28 00:24:36,071 : INFO : 50 batches submitted to accumulate stats from 3200 documents (275191 virtual) 2018-03-28 00:24:36,127 : INFO : 51 batches submitted to accumulate stats from 3264 documents (280913 virtual) 2018-03-28 00:24:36,178 : INFO : 52 batches submitted to accumulate stats from 3328 documents (286674 virtual) 2018-03-28 00:24:36,243 : INFO : 53 batches submitted to accumulate stats from 3392 documents (292159 virtual) 2018-03-28 00:24:36,313 : INFO : 54 batches submitted to accumulate stats from 3456 documents (297794 virtual) 2018-03-28 00:24:36,355 : INFO : 55 batches submitted to accumulate stats from 3520 documents (304132 virtual) 2018-03-28 00:24:36,395 : INFO : 56 batches submitted to accumulate stats from 3584 documents (308399 virtual) 2018-03-28 00:24:36,665 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:36,698 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:36,671 : INFO : accumulator serialized 2018-03-28 00:24:36,724 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:36,704 : INFO : accumulator serialized 2018-03-28 00:24:36,731 : INFO : accumulator serialized 2018-03-28 00:24:36,772 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:24:36,820 : INFO : accumulated word occurrence stats for 308567 virtual documents 2018-03-28 00:24:37,272 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_20 2018-03-28 00:24:37,274 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_20.expElogbeta.npy with mmap=None 2018-03-28 00:24:37,276 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:37,277 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:37,278 : INFO : setting ignored attribute state to None 2018-03-28 00:24:37,278 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_20 2018-03-28 00:24:37,279 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_20.state 2018-03-28 00:24:37,287 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_20.state 2018-03-28 00:24:37,299 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:24:37,338 : INFO : 1 batches submitted to accumulate stats from 64 documents (5664 virtual) 2018-03-28 00:24:37,349 : INFO : 2 batches submitted to accumulate stats from 128 documents (10419 virtual) 2018-03-28 00:24:37,355 : INFO : 3 batches submitted to accumulate stats from 192 documents (15719 virtual) 2018-03-28 00:24:37,362 : INFO : 4 batches submitted to accumulate stats from 256 documents (21000 virtual) 2018-03-28 00:24:37,370 : INFO : 5 batches submitted to accumulate stats from 320 documents (26529 virtual) 2018-03-28 00:24:37,377 : INFO : 6 batches submitted to accumulate stats from 384 documents (31837 virtual) 2018-03-28 00:24:37,541 : INFO : 7 batches submitted to accumulate stats from 448 documents (37364 virtual) 2018-03-28 00:24:37,548 : INFO : 8 batches submitted to accumulate stats from 512 documents (42466 virtual) 2018-03-28 00:24:37,572 : INFO : 9 batches submitted to accumulate stats from 576 documents (47807 virtual) 2018-03-28 00:24:37,722 : INFO : 10 batches submitted to accumulate stats from 640 documents (53215 virtual) 2018-03-28 00:24:37,746 : INFO : 11 batches submitted to accumulate stats from 704 documents (59102 virtual) 2018-03-28 00:24:37,773 : INFO : 12 batches submitted to accumulate stats from 768 documents (64353 virtual) 2018-03-28 00:24:37,916 : INFO : 13 batches submitted to accumulate stats from 832 documents (70392 virtual) 2018-03-28 00:24:37,933 : INFO : 14 batches submitted to accumulate stats from 896 documents (76633 virtual) 2018-03-28 00:24:37,975 : INFO : 15 batches submitted to accumulate stats from 960 documents (81603 virtual) 2018-03-28 00:24:38,119 : INFO : 16 batches submitted to accumulate stats from 1024 documents (87442 virtual) 2018-03-28 00:24:38,133 : INFO : 17 batches submitted to accumulate stats from 1088 documents (93180 virtual) 2018-03-28 00:24:38,157 : INFO : 18 batches submitted to accumulate stats from 1152 documents (98888 virtual) 2018-03-28 00:24:38,304 : INFO : 19 batches submitted to accumulate stats from 1216 documents (104392 virtual) 2018-03-28 00:24:38,338 : INFO : 20 batches submitted to accumulate stats from 1280 documents (109480 virtual) 2018-03-28 00:24:38,396 : INFO : 21 batches submitted to accumulate stats from 1344 documents (114883 virtual) 2018-03-28 00:24:38,501 : INFO : 22 batches submitted to accumulate stats from 1408 documents (119642 virtual) 2018-03-28 00:24:38,528 : INFO : 23 batches submitted to accumulate stats from 1472 documents (125571 virtual) 2018-03-28 00:24:38,621 : INFO : 24 batches submitted to accumulate stats from 1536 documents (131152 virtual) 2018-03-28 00:24:38,692 : INFO : 25 batches submitted to accumulate stats from 1600 documents (136924 virtual) 2018-03-28 00:24:38,725 : INFO : 26 batches submitted to accumulate stats from 1664 documents (142635 virtual) 2018-03-28 00:24:38,813 : INFO : 27 batches submitted to accumulate stats from 1728 documents (147591 virtual) 2018-03-28 00:24:38,859 : INFO : 28 batches submitted to accumulate stats from 1792 documents (152512 virtual) 2018-03-28 00:24:38,929 : INFO : 29 batches submitted to accumulate stats from 1856 documents (157743 virtual) 2018-03-28 00:24:39,026 : INFO : 30 batches submitted to accumulate stats from 1920 documents (163278 virtual) 2018-03-28 00:24:39,065 : INFO : 31 batches submitted to accumulate stats from 1984 documents (168786 virtual) 2018-03-28 00:24:39,123 : INFO : 32 batches submitted to accumulate stats from 2048 documents (174396 virtual) 2018-03-28 00:24:39,206 : INFO : 33 batches submitted to accumulate stats from 2112 documents (180502 virtual) 2018-03-28 00:24:39,255 : INFO : 34 batches submitted to accumulate stats from 2176 documents (185335 virtual) 2018-03-28 00:24:39,298 : INFO : 35 batches submitted to accumulate stats from 2240 documents (190704 virtual) 2018-03-28 00:24:39,411 : INFO : 36 batches submitted to accumulate stats from 2304 documents (196116 virtual) 2018-03-28 00:24:39,440 : INFO : 37 batches submitted to accumulate stats from 2368 documents (201616 virtual) 2018-03-28 00:24:39,479 : INFO : 38 batches submitted to accumulate stats from 2432 documents (207826 virtual) 2018-03-28 00:24:39,624 : INFO : 39 batches submitted to accumulate stats from 2496 documents (214002 virtual) 2018-03-28 00:24:39,635 : INFO : 40 batches submitted to accumulate stats from 2560 documents (218918 virtual) 2018-03-28 00:24:39,645 : INFO : 41 batches submitted to accumulate stats from 2624 documents (224350 virtual) 2018-03-28 00:24:39,809 : INFO : 42 batches submitted to accumulate stats from 2688 documents (230017 virtual) 2018-03-28 00:24:39,826 : INFO : 43 batches submitted to accumulate stats from 2752 documents (236012 virtual) 2018-03-28 00:24:39,869 : INFO : 44 batches submitted to accumulate stats from 2816 documents (241860 virtual) 2018-03-28 00:24:40,038 : INFO : 45 batches submitted to accumulate stats from 2880 documents (246509 virtual) 2018-03-28 00:24:40,066 : INFO : 46 batches submitted to accumulate stats from 2944 documents (252155 virtual) 2018-03-28 00:24:40,080 : INFO : 47 batches submitted to accumulate stats from 3008 documents (257643 virtual) 2018-03-28 00:24:40,249 : INFO : 48 batches submitted to accumulate stats from 3072 documents (263189 virtual) 2018-03-28 00:24:40,285 : INFO : 49 batches submitted to accumulate stats from 3136 documents (269010 virtual) 2018-03-28 00:24:40,291 : INFO : 50 batches submitted to accumulate stats from 3200 documents (274291 virtual) 2018-03-28 00:24:40,444 : INFO : 51 batches submitted to accumulate stats from 3264 documents (279765 virtual) 2018-03-28 00:24:40,464 : INFO : 52 batches submitted to accumulate stats from 3328 documents (284909 virtual) 2018-03-28 00:24:40,509 : INFO : 53 batches submitted to accumulate stats from 3392 documents (290712 virtual) 2018-03-28 00:24:40,646 : INFO : 54 batches submitted to accumulate stats from 3456 documents (296484 virtual) 2018-03-28 00:24:40,656 : INFO : 55 batches submitted to accumulate stats from 3520 documents (302671 virtual) 2018-03-28 00:24:40,703 : INFO : 56 batches submitted to accumulate stats from 3584 documents (307982 virtual) 2018-03-28 00:24:40,835 : INFO : 57 batches submitted to accumulate stats from 3648 documents (308374 virtual) 2018-03-28 00:24:41,052 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:41,060 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:41,087 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:41,067 : INFO : accumulator serialized 2018-03-28 00:24:41,058 : INFO : accumulator serialized 2018-03-28 00:24:41,096 : INFO : accumulator serialized 2018-03-28 00:24:41,170 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:24:41,256 : INFO : accumulated word occurrence stats for 308556 virtual documents 2018-03-28 00:24:41,857 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_25 2018-03-28 00:24:41,858 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-28 00:24:41,861 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:41,862 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:41,862 : INFO : setting ignored attribute state to None 2018-03-28 00:24:41,863 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_25 2018-03-28 00:24:41,864 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_25.state 2018-03-28 00:24:41,872 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_25.state 2018-03-28 00:24:41,885 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:24:41,921 : INFO : 1 batches submitted to accumulate stats from 64 documents (5664 virtual) 2018-03-28 00:24:41,929 : INFO : 2 batches submitted to accumulate stats from 128 documents (10419 virtual) 2018-03-28 00:24:41,936 : INFO : 3 batches submitted to accumulate stats from 192 documents (15719 virtual) 2018-03-28 00:24:41,943 : INFO : 4 batches submitted to accumulate stats from 256 documents (21000 virtual) 2018-03-28 00:24:41,951 : INFO : 5 batches submitted to accumulate stats from 320 documents (26636 virtual) 2018-03-28 00:24:41,958 : INFO : 6 batches submitted to accumulate stats from 384 documents (31832 virtual) 2018-03-28 00:24:42,117 : INFO : 7 batches submitted to accumulate stats from 448 documents (37352 virtual) 2018-03-28 00:24:42,153 : INFO : 8 batches submitted to accumulate stats from 512 documents (42377 virtual) 2018-03-28 00:24:42,161 : INFO : 9 batches submitted to accumulate stats from 576 documents (47646 virtual) 2018-03-28 00:24:42,322 : INFO : 10 batches submitted to accumulate stats from 640 documents (53138 virtual) 2018-03-28 00:24:42,347 : INFO : 11 batches submitted to accumulate stats from 704 documents (59040 virtual) 2018-03-28 00:24:42,378 : INFO : 12 batches submitted to accumulate stats from 768 documents (64343 virtual) 2018-03-28 00:24:42,526 : INFO : 13 batches submitted to accumulate stats from 832 documents (70303 virtual) 2018-03-28 00:24:42,554 : INFO : 14 batches submitted to accumulate stats from 896 documents (76515 virtual) 2018-03-28 00:24:42,571 : INFO : 15 batches submitted to accumulate stats from 960 documents (81501 virtual) 2018-03-28 00:24:42,718 : INFO : 16 batches submitted to accumulate stats from 1024 documents (87326 virtual) 2018-03-28 00:24:42,770 : INFO : 17 batches submitted to accumulate stats from 1088 documents (93024 virtual) 2018-03-28 00:24:42,784 : INFO : 18 batches submitted to accumulate stats from 1152 documents (98744 virtual) 2018-03-28 00:24:42,936 : INFO : 19 batches submitted to accumulate stats from 1216 documents (104348 virtual) 2018-03-28 00:24:42,956 : INFO : 20 batches submitted to accumulate stats from 1280 documents (109371 virtual) 2018-03-28 00:24:43,021 : INFO : 21 batches submitted to accumulate stats from 1344 documents (114824 virtual) 2018-03-28 00:24:43,144 : INFO : 22 batches submitted to accumulate stats from 1408 documents (119527 virtual) 2018-03-28 00:24:43,169 : INFO : 23 batches submitted to accumulate stats from 1472 documents (125432 virtual) 2018-03-28 00:24:43,243 : INFO : 24 batches submitted to accumulate stats from 1536 documents (131075 virtual) 2018-03-28 00:24:43,343 : INFO : 25 batches submitted to accumulate stats from 1600 documents (136911 virtual) 2018-03-28 00:24:43,371 : INFO : 26 batches submitted to accumulate stats from 1664 documents (142471 virtual) 2018-03-28 00:24:43,444 : INFO : 27 batches submitted to accumulate stats from 1728 documents (147539 virtual) 2018-03-28 00:24:43,530 : INFO : 28 batches submitted to accumulate stats from 1792 documents (152468 virtual) 2018-03-28 00:24:43,593 : INFO : 29 batches submitted to accumulate stats from 1856 documents (157506 virtual) 2018-03-28 00:24:43,664 : INFO : 30 batches submitted to accumulate stats from 1920 documents (163157 virtual) 2018-03-28 00:24:43,761 : INFO : 31 batches submitted to accumulate stats from 1984 documents (168651 virtual) 2018-03-28 00:24:43,798 : INFO : 32 batches submitted to accumulate stats from 2048 documents (174259 virtual) 2018-03-28 00:24:43,866 : INFO : 33 batches submitted to accumulate stats from 2112 documents (180413 virtual) 2018-03-28 00:24:43,943 : INFO : 34 batches submitted to accumulate stats from 2176 documents (185264 virtual) 2018-03-28 00:24:43,987 : INFO : 35 batches submitted to accumulate stats from 2240 documents (190566 virtual) 2018-03-28 00:24:44,064 : INFO : 36 batches submitted to accumulate stats from 2304 documents (196007 virtual) 2018-03-28 00:24:44,135 : INFO : 37 batches submitted to accumulate stats from 2368 documents (201519 virtual) 2018-03-28 00:24:44,203 : INFO : 38 batches submitted to accumulate stats from 2432 documents (207602 virtual) 2018-03-28 00:24:44,241 : INFO : 39 batches submitted to accumulate stats from 2496 documents (213857 virtual) 2018-03-28 00:24:44,313 : INFO : 40 batches submitted to accumulate stats from 2560 documents (218799 virtual) 2018-03-28 00:24:44,407 : INFO : 41 batches submitted to accumulate stats from 2624 documents (224227 virtual) 2018-03-28 00:24:44,449 : INFO : 42 batches submitted to accumulate stats from 2688 documents (229886 virtual) 2018-03-28 00:24:44,546 : INFO : 43 batches submitted to accumulate stats from 2752 documents (235780 virtual) 2018-03-28 00:24:44,632 : INFO : 44 batches submitted to accumulate stats from 2816 documents (241707 virtual) 2018-03-28 00:24:44,710 : INFO : 45 batches submitted to accumulate stats from 2880 documents (246368 virtual) 2018-03-28 00:24:44,748 : INFO : 46 batches submitted to accumulate stats from 2944 documents (251922 virtual) 2018-03-28 00:24:44,901 : INFO : 47 batches submitted to accumulate stats from 3008 documents (257323 virtual) 2018-03-28 00:24:44,938 : INFO : 48 batches submitted to accumulate stats from 3072 documents (262872 virtual) 2018-03-28 00:24:45,020 : INFO : 49 batches submitted to accumulate stats from 3136 documents (268678 virtual) 2018-03-28 00:24:45,110 : INFO : 50 batches submitted to accumulate stats from 3200 documents (274036 virtual) 2018-03-28 00:24:45,125 : INFO : 51 batches submitted to accumulate stats from 3264 documents (279299 virtual) 2018-03-28 00:24:45,240 : INFO : 52 batches submitted to accumulate stats from 3328 documents (284511 virtual) 2018-03-28 00:24:45,348 : INFO : 53 batches submitted to accumulate stats from 3392 documents (290305 virtual) 2018-03-28 00:24:45,369 : INFO : 54 batches submitted to accumulate stats from 3456 documents (296012 virtual) 2018-03-28 00:24:45,503 : INFO : 55 batches submitted to accumulate stats from 3520 documents (302320 virtual) 2018-03-28 00:24:45,573 : INFO : 56 batches submitted to accumulate stats from 3584 documents (307574 virtual) 2018-03-28 00:24:45,575 : INFO : 57 batches submitted to accumulate stats from 3648 documents (308458 virtual) 2018-03-28 00:24:45,831 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:45,837 : INFO : accumulator serialized 2018-03-28 00:24:45,928 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:45,948 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:45,935 : INFO : accumulator serialized 2018-03-28 00:24:45,953 : INFO : accumulator serialized 2018-03-28 00:24:46,027 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:24:46,137 : INFO : accumulated word occurrence stats for 308656 virtual documents 2018-03-28 00:24:46,882 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_30 2018-03-28 00:24:46,884 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-28 00:24:46,887 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:46,887 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:46,888 : INFO : setting ignored attribute state to None 2018-03-28 00:24:46,888 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_30 2018-03-28 00:24:46,889 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_30.state 2018-03-28 00:24:46,898 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_30.state 2018-03-28 00:24:46,913 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:24:46,947 : INFO : 1 batches submitted to accumulate stats from 64 documents (5664 virtual) 2018-03-28 00:24:46,956 : INFO : 2 batches submitted to accumulate stats from 128 documents (10419 virtual) 2018-03-28 00:24:46,963 : INFO : 3 batches submitted to accumulate stats from 192 documents (15719 virtual) 2018-03-28 00:24:46,970 : INFO : 4 batches submitted to accumulate stats from 256 documents (21000 virtual) 2018-03-28 00:24:46,978 : INFO : 5 batches submitted to accumulate stats from 320 documents (26529 virtual) 2018-03-28 00:24:46,985 : INFO : 6 batches submitted to accumulate stats from 384 documents (31703 virtual) 2018-03-28 00:24:47,171 : INFO : 7 batches submitted to accumulate stats from 448 documents (37361 virtual) 2018-03-28 00:24:47,193 : INFO : 8 batches submitted to accumulate stats from 512 documents (42386 virtual) 2018-03-28 00:24:47,203 : INFO : 9 batches submitted to accumulate stats from 576 documents (47655 virtual) 2018-03-28 00:24:47,427 : INFO : 10 batches submitted to accumulate stats from 640 documents (53147 virtual) 2018-03-28 00:24:47,439 : INFO : 11 batches submitted to accumulate stats from 704 documents (59049 virtual) 2018-03-28 00:24:47,474 : INFO : 12 batches submitted to accumulate stats from 768 documents (64352 virtual) 2018-03-28 00:24:47,635 : INFO : 13 batches submitted to accumulate stats from 832 documents (70312 virtual) 2018-03-28 00:24:47,645 : INFO : 14 batches submitted to accumulate stats from 896 documents (76524 virtual) 2018-03-28 00:24:47,699 : INFO : 15 batches submitted to accumulate stats from 960 documents (81510 virtual) 2018-03-28 00:24:47,855 : INFO : 16 batches submitted to accumulate stats from 1024 documents (87335 virtual) 2018-03-28 00:24:47,864 : INFO : 17 batches submitted to accumulate stats from 1088 documents (93033 virtual) 2018-03-28 00:24:47,924 : INFO : 18 batches submitted to accumulate stats from 1152 documents (98753 virtual) 2018-03-28 00:24:48,091 : INFO : 19 batches submitted to accumulate stats from 1216 documents (104357 virtual) 2018-03-28 00:24:48,100 : INFO : 20 batches submitted to accumulate stats from 1280 documents (109380 virtual) 2018-03-28 00:24:48,145 : INFO : 21 batches submitted to accumulate stats from 1344 documents (114833 virtual) 2018-03-28 00:24:48,290 : INFO : 22 batches submitted to accumulate stats from 1408 documents (119536 virtual) 2018-03-28 00:24:48,339 : INFO : 23 batches submitted to accumulate stats from 1472 documents (125441 virtual) 2018-03-28 00:24:48,395 : INFO : 24 batches submitted to accumulate stats from 1536 documents (131084 virtual) 2018-03-28 00:24:48,532 : INFO : 25 batches submitted to accumulate stats from 1600 documents (136920 virtual) 2018-03-28 00:24:48,546 : INFO : 26 batches submitted to accumulate stats from 1664 documents (142480 virtual) 2018-03-28 00:24:48,616 : INFO : 27 batches submitted to accumulate stats from 1728 documents (147548 virtual) 2018-03-28 00:24:48,730 : INFO : 28 batches submitted to accumulate stats from 1792 documents (152477 virtual) 2018-03-28 00:24:48,797 : INFO : 29 batches submitted to accumulate stats from 1856 documents (157515 virtual) 2018-03-28 00:24:48,819 : INFO : 30 batches submitted to accumulate stats from 1920 documents (163023 virtual) 2018-03-28 00:24:48,955 : INFO : 31 batches submitted to accumulate stats from 1984 documents (168434 virtual) 2018-03-28 00:24:49,008 : INFO : 32 batches submitted to accumulate stats from 2048 documents (174247 virtual) 2018-03-28 00:24:49,050 : INFO : 33 batches submitted to accumulate stats from 2112 documents (180258 virtual) 2018-03-28 00:24:49,149 : INFO : 34 batches submitted to accumulate stats from 2176 documents (185262 virtual) 2018-03-28 00:24:49,248 : INFO : 35 batches submitted to accumulate stats from 2240 documents (190501 virtual) 2018-03-28 00:24:49,254 : INFO : 36 batches submitted to accumulate stats from 2304 documents (195924 virtual) 2018-03-28 00:24:49,357 : INFO : 37 batches submitted to accumulate stats from 2368 documents (201422 virtual) 2018-03-28 00:24:49,494 : INFO : 38 batches submitted to accumulate stats from 2432 documents (207564 virtual) 2018-03-28 00:24:49,501 : INFO : 39 batches submitted to accumulate stats from 2496 documents (213811 virtual) 2018-03-28 00:24:49,553 : INFO : 40 batches submitted to accumulate stats from 2560 documents (218707 virtual) 2018-03-28 00:24:49,711 : INFO : 41 batches submitted to accumulate stats from 2624 documents (224207 virtual) 2018-03-28 00:24:49,722 : INFO : 42 batches submitted to accumulate stats from 2688 documents (229874 virtual) 2018-03-28 00:24:49,771 : INFO : 43 batches submitted to accumulate stats from 2752 documents (235789 virtual) 2018-03-28 00:24:49,961 : INFO : 44 batches submitted to accumulate stats from 2816 documents (241716 virtual) 2018-03-28 00:24:49,967 : INFO : 45 batches submitted to accumulate stats from 2880 documents (246377 virtual) 2018-03-28 00:24:49,980 : INFO : 46 batches submitted to accumulate stats from 2944 documents (251931 virtual) 2018-03-28 00:24:50,179 : INFO : 47 batches submitted to accumulate stats from 3008 documents (257332 virtual) 2018-03-28 00:24:50,186 : INFO : 48 batches submitted to accumulate stats from 3072 documents (262813 virtual) 2018-03-28 00:24:50,204 : INFO : 49 batches submitted to accumulate stats from 3136 documents (268666 virtual) 2018-03-28 00:24:50,389 : INFO : 50 batches submitted to accumulate stats from 3200 documents (274015 virtual) 2018-03-28 00:24:50,405 : INFO : 51 batches submitted to accumulate stats from 3264 documents (279231 virtual) 2018-03-28 00:24:50,427 : INFO : 52 batches submitted to accumulate stats from 3328 documents (284428 virtual) 2018-03-28 00:24:50,606 : INFO : 53 batches submitted to accumulate stats from 3392 documents (290241 virtual) 2018-03-28 00:24:50,630 : INFO : 54 batches submitted to accumulate stats from 3456 documents (295925 virtual) 2018-03-28 00:24:50,680 : INFO : 55 batches submitted to accumulate stats from 3520 documents (302155 virtual) 2018-03-28 00:24:50,810 : INFO : 56 batches submitted to accumulate stats from 3584 documents (307589 virtual) 2018-03-28 00:24:50,862 : INFO : 57 batches submitted to accumulate stats from 3648 documents (308473 virtual) 2018-03-28 00:24:51,114 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:51,116 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:51,124 : INFO : accumulator serialized 2018-03-28 00:24:51,120 : INFO : accumulator serialized 2018-03-28 00:24:51,277 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:51,282 : INFO : accumulator serialized 2018-03-28 00:24:51,360 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:24:51,505 : INFO : accumulated word occurrence stats for 308662 virtual documents 2018-03-28 00:24:52,374 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_35 2018-03-28 00:24:52,375 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_35.expElogbeta.npy with mmap=None 2018-03-28 00:24:52,378 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:52,379 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:52,379 : INFO : setting ignored attribute state to None 2018-03-28 00:24:52,381 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_35 2018-03-28 00:24:52,381 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_35.state 2018-03-28 00:24:52,392 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_35.state 2018-03-28 00:24:52,409 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:24:52,445 : INFO : 1 batches submitted to accumulate stats from 64 documents (5664 virtual) 2018-03-28 00:24:52,452 : INFO : 2 batches submitted to accumulate stats from 128 documents (10419 virtual) 2018-03-28 00:24:52,460 : INFO : 3 batches submitted to accumulate stats from 192 documents (15719 virtual) 2018-03-28 00:24:52,467 : INFO : 4 batches submitted to accumulate stats from 256 documents (21000 virtual) 2018-03-28 00:24:52,475 : INFO : 5 batches submitted to accumulate stats from 320 documents (26464 virtual) 2018-03-28 00:24:52,482 : INFO : 6 batches submitted to accumulate stats from 384 documents (31708 virtual) 2018-03-28 00:24:52,642 : INFO : 7 batches submitted to accumulate stats from 448 documents (37371 virtual) 2018-03-28 00:24:52,708 : INFO : 8 batches submitted to accumulate stats from 512 documents (42473 virtual) 2018-03-28 00:24:52,725 : INFO : 9 batches submitted to accumulate stats from 576 documents (47814 virtual) 2018-03-28 00:24:52,872 : INFO : 10 batches submitted to accumulate stats from 640 documents (53222 virtual) 2018-03-28 00:24:52,914 : INFO : 11 batches submitted to accumulate stats from 704 documents (59109 virtual) 2018-03-28 00:24:52,952 : INFO : 12 batches submitted to accumulate stats from 768 documents (64360 virtual) 2018-03-28 00:24:53,113 : INFO : 13 batches submitted to accumulate stats from 832 documents (70399 virtual) 2018-03-28 00:24:53,139 : INFO : 14 batches submitted to accumulate stats from 896 documents (76398 virtual) 2018-03-28 00:24:53,189 : INFO : 15 batches submitted to accumulate stats from 960 documents (81446 virtual) 2018-03-28 00:24:53,339 : INFO : 16 batches submitted to accumulate stats from 1024 documents (87191 virtual) 2018-03-28 00:24:53,396 : INFO : 17 batches submitted to accumulate stats from 1088 documents (92872 virtual) 2018-03-28 00:24:53,408 : INFO : 18 batches submitted to accumulate stats from 1152 documents (98568 virtual) 2018-03-28 00:24:53,608 : INFO : 19 batches submitted to accumulate stats from 1216 documents (104059 virtual) 2018-03-28 00:24:53,624 : INFO : 20 batches submitted to accumulate stats from 1280 documents (109182 virtual) 2018-03-28 00:24:53,632 : INFO : 21 batches submitted to accumulate stats from 1344 documents (114494 virtual) 2018-03-28 00:24:53,822 : INFO : 22 batches submitted to accumulate stats from 1408 documents (119431 virtual) 2018-03-28 00:24:53,872 : INFO : 23 batches submitted to accumulate stats from 1472 documents (125124 virtual) 2018-03-28 00:24:53,879 : INFO : 24 batches submitted to accumulate stats from 1536 documents (130914 virtual) 2018-03-28 00:24:54,087 : INFO : 25 batches submitted to accumulate stats from 1600 documents (136736 virtual) 2018-03-28 00:24:54,113 : INFO : 26 batches submitted to accumulate stats from 1664 documents (142384 virtual) 2018-03-28 00:24:54,127 : INFO : 27 batches submitted to accumulate stats from 1728 documents (147311 virtual) 2018-03-28 00:24:54,312 : INFO : 28 batches submitted to accumulate stats from 1792 documents (152294 virtual) 2018-03-28 00:24:54,354 : INFO : 29 batches submitted to accumulate stats from 1856 documents (157255 virtual) 2018-03-28 00:24:54,388 : INFO : 30 batches submitted to accumulate stats from 1920 documents (162594 virtual) 2018-03-28 00:24:54,567 : INFO : 31 batches submitted to accumulate stats from 1984 documents (168004 virtual) 2018-03-28 00:24:54,575 : INFO : 32 batches submitted to accumulate stats from 2048 documents (173909 virtual) 2018-03-28 00:24:54,616 : INFO : 33 batches submitted to accumulate stats from 2112 documents (179932 virtual) 2018-03-28 00:24:54,779 : INFO : 34 batches submitted to accumulate stats from 2176 documents (185090 virtual) 2018-03-28 00:24:54,789 : INFO : 35 batches submitted to accumulate stats from 2240 documents (190308 virtual) 2018-03-28 00:24:54,828 : INFO : 36 batches submitted to accumulate stats from 2304 documents (195522 virtual) 2018-03-28 00:24:54,973 : INFO : 37 batches submitted to accumulate stats from 2368 documents (201004 virtual) 2018-03-28 00:24:55,046 : INFO : 38 batches submitted to accumulate stats from 2432 documents (207087 virtual) 2018-03-28 00:24:55,060 : INFO : 39 batches submitted to accumulate stats from 2496 documents (213279 virtual) 2018-03-28 00:24:55,191 : INFO : 40 batches submitted to accumulate stats from 2560 documents (218336 virtual) 2018-03-28 00:24:55,255 : INFO : 41 batches submitted to accumulate stats from 2624 documents (223652 virtual) 2018-03-28 00:24:55,269 : INFO : 42 batches submitted to accumulate stats from 2688 documents (229365 virtual) 2018-03-28 00:24:55,425 : INFO : 43 batches submitted to accumulate stats from 2752 documents (235217 virtual) 2018-03-28 00:24:55,499 : INFO : 44 batches submitted to accumulate stats from 2816 documents (241275 virtual) 2018-03-28 00:24:55,550 : INFO : 45 batches submitted to accumulate stats from 2880 documents (246155 virtual) 2018-03-28 00:24:55,630 : INFO : 46 batches submitted to accumulate stats from 2944 documents (251513 virtual) 2018-03-28 00:24:55,692 : INFO : 47 batches submitted to accumulate stats from 3008 documents (256728 virtual) 2018-03-28 00:24:55,793 : INFO : 48 batches submitted to accumulate stats from 3072 documents (262160 virtual) 2018-03-28 00:24:55,852 : INFO : 49 batches submitted to accumulate stats from 3136 documents (268217 virtual) 2018-03-28 00:24:55,932 : INFO : 50 batches submitted to accumulate stats from 3200 documents (273593 virtual) 2018-03-28 00:24:55,995 : INFO : 51 batches submitted to accumulate stats from 3264 documents (278537 virtual) 2018-03-28 00:24:56,092 : INFO : 52 batches submitted to accumulate stats from 3328 documents (283906 virtual) 2018-03-28 00:24:56,149 : INFO : 53 batches submitted to accumulate stats from 3392 documents (290031 virtual) 2018-03-28 00:24:56,229 : INFO : 54 batches submitted to accumulate stats from 3456 documents (295440 virtual) 2018-03-28 00:24:56,353 : INFO : 55 batches submitted to accumulate stats from 3520 documents (301390 virtual) 2018-03-28 00:24:56,361 : INFO : 56 batches submitted to accumulate stats from 3584 documents (306974 virtual) 2018-03-28 00:24:56,417 : INFO : 57 batches submitted to accumulate stats from 3648 documents (308674 virtual) 2018-03-28 00:24:56,718 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:56,791 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:56,724 : INFO : accumulator serialized 2018-03-28 00:24:56,824 : INFO : serializing accumulator to return to master... 2018-03-28 00:24:56,798 : INFO : accumulator serialized 2018-03-28 00:24:56,831 : INFO : accumulator serialized 2018-03-28 00:24:56,924 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:24:57,087 : INFO : accumulated word occurrence stats for 308859 virtual documents 2018-03-28 00:24:58,122 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_40 2018-03-28 00:24:58,124 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-28 00:24:58,127 : INFO : setting ignored attribute id2word to None 2018-03-28 00:24:58,128 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:24:58,129 : INFO : setting ignored attribute state to None 2018-03-28 00:24:58,129 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_40 2018-03-28 00:24:58,130 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_40.state 2018-03-28 00:24:58,143 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_40.state 2018-03-28 00:24:58,163 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:24:58,202 : INFO : 1 batches submitted to accumulate stats from 64 documents (5664 virtual) 2018-03-28 00:24:58,209 : INFO : 2 batches submitted to accumulate stats from 128 documents (10419 virtual) 2018-03-28 00:24:58,216 : INFO : 3 batches submitted to accumulate stats from 192 documents (15719 virtual) 2018-03-28 00:24:58,225 : INFO : 4 batches submitted to accumulate stats from 256 documents (21000 virtual) 2018-03-28 00:24:58,233 : INFO : 5 batches submitted to accumulate stats from 320 documents (26464 virtual) 2018-03-28 00:24:58,241 : INFO : 6 batches submitted to accumulate stats from 384 documents (31629 virtual) 2018-03-28 00:24:58,438 : INFO : 7 batches submitted to accumulate stats from 448 documents (37187 virtual) 2018-03-28 00:24:58,480 : INFO : 8 batches submitted to accumulate stats from 512 documents (42347 virtual) 2018-03-28 00:24:58,487 : INFO : 9 batches submitted to accumulate stats from 576 documents (47579 virtual) 2018-03-28 00:24:58,663 : INFO : 10 batches submitted to accumulate stats from 640 documents (53119 virtual) 2018-03-28 00:24:58,728 : INFO : 11 batches submitted to accumulate stats from 704 documents (59014 virtual) 2018-03-28 00:24:58,747 : INFO : 12 batches submitted to accumulate stats from 768 documents (64333 virtual) 2018-03-28 00:24:58,901 : INFO : 13 batches submitted to accumulate stats from 832 documents (70217 virtual) 2018-03-28 00:24:58,967 : INFO : 14 batches submitted to accumulate stats from 896 documents (76278 virtual) 2018-03-28 00:24:58,990 : INFO : 15 batches submitted to accumulate stats from 960 documents (81280 virtual) 2018-03-28 00:24:59,139 : INFO : 16 batches submitted to accumulate stats from 1024 documents (86955 virtual) 2018-03-28 00:24:59,213 : INFO : 17 batches submitted to accumulate stats from 1088 documents (92598 virtual) 2018-03-28 00:24:59,236 : INFO : 18 batches submitted to accumulate stats from 1152 documents (98386 virtual) 2018-03-28 00:24:59,396 : INFO : 19 batches submitted to accumulate stats from 1216 documents (103916 virtual) 2018-03-28 00:24:59,455 : INFO : 20 batches submitted to accumulate stats from 1280 documents (108948 virtual) 2018-03-28 00:24:59,490 : INFO : 21 batches submitted to accumulate stats from 1344 documents (114259 virtual) 2018-03-28 00:24:59,640 : INFO : 22 batches submitted to accumulate stats from 1408 documents (119338 virtual) 2018-03-28 00:24:59,711 : INFO : 23 batches submitted to accumulate stats from 1472 documents (124909 virtual) 2018-03-28 00:24:59,794 : INFO : 24 batches submitted to accumulate stats from 1536 documents (130686 virtual) 2018-03-28 00:24:59,919 : INFO : 25 batches submitted to accumulate stats from 1600 documents (136555 virtual) 2018-03-28 00:24:59,999 : INFO : 26 batches submitted to accumulate stats from 1664 documents (142139 virtual) 2018-03-28 00:25:00,031 : INFO : 27 batches submitted to accumulate stats from 1728 documents (147021 virtual) 2018-03-28 00:25:00,159 : INFO : 28 batches submitted to accumulate stats from 1792 documents (152035 virtual) 2018-03-28 00:25:00,220 : INFO : 29 batches submitted to accumulate stats from 1856 documents (157004 virtual) 2018-03-28 00:25:00,292 : INFO : 30 batches submitted to accumulate stats from 1920 documents (162463 virtual) 2018-03-28 00:25:00,404 : INFO : 31 batches submitted to accumulate stats from 1984 documents (167798 virtual) 2018-03-28 00:25:00,498 : INFO : 32 batches submitted to accumulate stats from 2048 documents (173647 virtual) 2018-03-28 00:25:00,543 : INFO : 33 batches submitted to accumulate stats from 2112 documents (179632 virtual) 2018-03-28 00:25:00,654 : INFO : 34 batches submitted to accumulate stats from 2176 documents (184747 virtual) 2018-03-28 00:25:00,715 : INFO : 35 batches submitted to accumulate stats from 2240 documents (189960 virtual) 2018-03-28 00:25:00,789 : INFO : 36 batches submitted to accumulate stats from 2304 documents (195310 virtual) 2018-03-28 00:25:00,891 : INFO : 37 batches submitted to accumulate stats from 2368 documents (200572 virtual) 2018-03-28 00:25:00,960 : INFO : 38 batches submitted to accumulate stats from 2432 documents (206613 virtual) 2018-03-28 00:25:01,046 : INFO : 39 batches submitted to accumulate stats from 2496 documents (212418 virtual) 2018-03-28 00:25:01,127 : INFO : 40 batches submitted to accumulate stats from 2560 documents (217855 virtual) 2018-03-28 00:25:01,183 : INFO : 41 batches submitted to accumulate stats from 2624 documents (223217 virtual) 2018-03-28 00:25:01,296 : INFO : 42 batches submitted to accumulate stats from 2688 documents (229087 virtual) 2018-03-28 00:25:01,364 : INFO : 43 batches submitted to accumulate stats from 2752 documents (234493 virtual) 2018-03-28 00:25:01,437 : INFO : 44 batches submitted to accumulate stats from 2816 documents (240703 virtual) 2018-03-28 00:25:01,566 : INFO : 45 batches submitted to accumulate stats from 2880 documents (245736 virtual) 2018-03-28 00:25:01,610 : INFO : 46 batches submitted to accumulate stats from 2944 documents (250934 virtual) 2018-03-28 00:25:01,684 : INFO : 47 batches submitted to accumulate stats from 3008 documents (256363 virtual) 2018-03-28 00:25:01,801 : INFO : 48 batches submitted to accumulate stats from 3072 documents (261469 virtual) 2018-03-28 00:25:01,843 : INFO : 49 batches submitted to accumulate stats from 3136 documents (267727 virtual) 2018-03-28 00:25:01,957 : INFO : 50 batches submitted to accumulate stats from 3200 documents (272886 virtual) 2018-03-28 00:25:02,024 : INFO : 51 batches submitted to accumulate stats from 3264 documents (277857 virtual) 2018-03-28 00:25:02,070 : INFO : 52 batches submitted to accumulate stats from 3328 documents (283361 virtual) 2018-03-28 00:25:02,186 : INFO : 53 batches submitted to accumulate stats from 3392 documents (289483 virtual) 2018-03-28 00:25:02,263 : INFO : 54 batches submitted to accumulate stats from 3456 documents (294839 virtual) 2018-03-28 00:25:02,391 : INFO : 55 batches submitted to accumulate stats from 3520 documents (300772 virtual) 2018-03-28 00:25:02,479 : INFO : 56 batches submitted to accumulate stats from 3584 documents (306550 virtual) 2018-03-28 00:25:02,520 : INFO : 57 batches submitted to accumulate stats from 3648 documents (308663 virtual) 2018-03-28 00:25:02,857 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:02,909 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:02,989 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:02,916 : INFO : accumulator serialized 2018-03-28 00:25:02,997 : INFO : accumulator serialized 2018-03-28 00:25:02,862 : INFO : accumulator serialized 2018-03-28 00:25:04,400 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:25:04,689 : INFO : accumulated word occurrence stats for 308874 virtual documents 2018-03-28 00:25:05,863 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_50 2018-03-28 00:25:05,864 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_50.expElogbeta.npy with mmap=None 2018-03-28 00:25:05,868 : INFO : setting ignored attribute id2word to None 2018-03-28 00:25:05,869 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:25:05,869 : INFO : setting ignored attribute state to None 2018-03-28 00:25:05,870 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_50 2018-03-28 00:25:05,871 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_50.state 2018-03-28 00:25:05,886 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_50.state 2018-03-28 00:25:05,910 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:25:05,954 : INFO : 1 batches submitted to accumulate stats from 64 documents (5664 virtual) 2018-03-28 00:25:05,962 : INFO : 2 batches submitted to accumulate stats from 128 documents (10419 virtual) 2018-03-28 00:25:05,968 : INFO : 3 batches submitted to accumulate stats from 192 documents (15719 virtual) 2018-03-28 00:25:05,977 : INFO : 4 batches submitted to accumulate stats from 256 documents (21000 virtual) 2018-03-28 00:25:05,985 : INFO : 5 batches submitted to accumulate stats from 320 documents (26529 virtual) 2018-03-28 00:25:05,992 : INFO : 6 batches submitted to accumulate stats from 384 documents (31703 virtual) 2018-03-28 00:25:06,221 : INFO : 7 batches submitted to accumulate stats from 448 documents (37180 virtual) 2018-03-28 00:25:06,252 : INFO : 8 batches submitted to accumulate stats from 512 documents (42303 virtual) 2018-03-28 00:25:06,275 : INFO : 9 batches submitted to accumulate stats from 576 documents (47476 virtual) 2018-03-28 00:25:06,486 : INFO : 10 batches submitted to accumulate stats from 640 documents (52956 virtual) 2018-03-28 00:25:06,497 : INFO : 11 batches submitted to accumulate stats from 704 documents (58907 virtual) 2018-03-28 00:25:06,561 : INFO : 12 batches submitted to accumulate stats from 768 documents (64320 virtual) 2018-03-28 00:25:06,790 : INFO : 13 batches submitted to accumulate stats from 832 documents (70077 virtual) 2018-03-28 00:25:06,800 : INFO : 14 batches submitted to accumulate stats from 896 documents (76170 virtual) 2018-03-28 00:25:06,855 : INFO : 15 batches submitted to accumulate stats from 960 documents (81264 virtual) 2018-03-28 00:25:07,042 : INFO : 16 batches submitted to accumulate stats from 1024 documents (86854 virtual) 2018-03-28 00:25:07,099 : INFO : 17 batches submitted to accumulate stats from 1088 documents (92560 virtual) 2018-03-28 00:25:07,131 : INFO : 18 batches submitted to accumulate stats from 1152 documents (98362 virtual) 2018-03-28 00:25:07,305 : INFO : 19 batches submitted to accumulate stats from 1216 documents (103856 virtual) 2018-03-28 00:25:07,410 : INFO : 20 batches submitted to accumulate stats from 1280 documents (108843 virtual) 2018-03-28 00:25:07,426 : INFO : 21 batches submitted to accumulate stats from 1344 documents (114128 virtual) 2018-03-28 00:25:07,580 : INFO : 22 batches submitted to accumulate stats from 1408 documents (119207 virtual) 2018-03-28 00:25:07,689 : INFO : 23 batches submitted to accumulate stats from 1472 documents (124710 virtual) 2018-03-28 00:25:07,727 : INFO : 24 batches submitted to accumulate stats from 1536 documents (130521 virtual) 2018-03-28 00:25:07,853 : INFO : 25 batches submitted to accumulate stats from 1600 documents (136395 virtual) 2018-03-28 00:25:07,934 : INFO : 26 batches submitted to accumulate stats from 1664 documents (142064 virtual) 2018-03-28 00:25:07,985 : INFO : 27 batches submitted to accumulate stats from 1728 documents (146798 virtual) 2018-03-28 00:25:08,127 : INFO : 28 batches submitted to accumulate stats from 1792 documents (151967 virtual) 2018-03-28 00:25:08,149 : INFO : 29 batches submitted to accumulate stats from 1856 documents (156931 virtual) 2018-03-28 00:25:08,278 : INFO : 30 batches submitted to accumulate stats from 1920 documents (162120 virtual) 2018-03-28 00:25:08,405 : INFO : 31 batches submitted to accumulate stats from 1984 documents (167657 virtual) 2018-03-28 00:25:08,418 : INFO : 32 batches submitted to accumulate stats from 2048 documents (173392 virtual) 2018-03-28 00:25:08,504 : INFO : 33 batches submitted to accumulate stats from 2112 documents (179271 virtual) 2018-03-28 00:25:08,634 : INFO : 34 batches submitted to accumulate stats from 2176 documents (184456 virtual) 2018-03-28 00:25:08,660 : INFO : 35 batches submitted to accumulate stats from 2240 documents (189712 virtual) 2018-03-28 00:25:08,766 : INFO : 36 batches submitted to accumulate stats from 2304 documents (194968 virtual) 2018-03-28 00:25:08,869 : INFO : 37 batches submitted to accumulate stats from 2368 documents (200416 virtual) 2018-03-28 00:25:08,925 : INFO : 38 batches submitted to accumulate stats from 2432 documents (206607 virtual) 2018-03-28 00:25:09,055 : INFO : 39 batches submitted to accumulate stats from 2496 documents (212256 virtual) 2018-03-28 00:25:09,128 : INFO : 40 batches submitted to accumulate stats from 2560 documents (217726 virtual) 2018-03-28 00:25:09,193 : INFO : 41 batches submitted to accumulate stats from 2624 documents (223132 virtual) 2018-03-28 00:25:09,252 : INFO : 42 batches submitted to accumulate stats from 2688 documents (228996 virtual) 2018-03-28 00:25:09,413 : INFO : 43 batches submitted to accumulate stats from 2752 documents (234484 virtual) 2018-03-28 00:25:09,494 : INFO : 44 batches submitted to accumulate stats from 2816 documents (240694 virtual) 2018-03-28 00:25:09,501 : INFO : 45 batches submitted to accumulate stats from 2880 documents (245727 virtual) 2018-03-28 00:25:09,701 : INFO : 46 batches submitted to accumulate stats from 2944 documents (250925 virtual) 2018-03-28 00:25:09,734 : INFO : 47 batches submitted to accumulate stats from 3008 documents (256354 virtual) 2018-03-28 00:25:09,742 : INFO : 48 batches submitted to accumulate stats from 3072 documents (261509 virtual) 2018-03-28 00:25:09,910 : INFO : 49 batches submitted to accumulate stats from 3136 documents (267781 virtual) 2018-03-28 00:25:09,969 : INFO : 50 batches submitted to accumulate stats from 3200 documents (272961 virtual) 2018-03-28 00:25:10,112 : INFO : 51 batches submitted to accumulate stats from 3264 documents (278022 virtual) 2018-03-28 00:25:10,165 : INFO : 52 batches submitted to accumulate stats from 3328 documents (283416 virtual) 2018-03-28 00:25:10,194 : INFO : 53 batches submitted to accumulate stats from 3392 documents (289548 virtual) 2018-03-28 00:25:10,347 : INFO : 54 batches submitted to accumulate stats from 3456 documents (294850 virtual) 2018-03-28 00:25:10,445 : INFO : 55 batches submitted to accumulate stats from 3520 documents (300906 virtual) 2018-03-28 00:25:10,452 : INFO : 56 batches submitted to accumulate stats from 3584 documents (306547 virtual) 2018-03-28 00:25:10,611 : INFO : 57 batches submitted to accumulate stats from 3648 documents (308658 virtual) 2018-03-28 00:25:11,003 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:11,010 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:11,011 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:11,010 : INFO : accumulator serialized 2018-03-28 00:25:11,018 : INFO : accumulator serialized 2018-03-28 00:25:11,018 : INFO : accumulator serialized 2018-03-28 00:25:11,248 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:25:11,559 : INFO : accumulated word occurrence stats for 308867 virtual documents
In [26]:
tmrest_meta_score['c_uci'] = tmrest_meta_score.apply(lambda row: get_score(row.num_topic, dict_tmrest_meta, texts=text_tmrest_meta, corpus=corpus_tmrest_meta, model_dir='../src/tmrest_meta/',
coherence='c_uci'),
axis=1)
2018-03-28 00:25:12,999 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_10 2018-03-28 00:25:13,002 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-28 00:25:13,003 : INFO : setting ignored attribute id2word to None 2018-03-28 00:25:13,004 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:25:13,004 : INFO : setting ignored attribute state to None 2018-03-28 00:25:13,005 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_10 2018-03-28 00:25:13,005 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_10.state 2018-03-28 00:25:13,010 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_10.state 2018-03-28 00:25:13,017 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:25:13,051 : INFO : 1 batches submitted to accumulate stats from 64 documents (5664 virtual) 2018-03-28 00:25:13,060 : INFO : 2 batches submitted to accumulate stats from 128 documents (10490 virtual) 2018-03-28 00:25:13,067 : INFO : 3 batches submitted to accumulate stats from 192 documents (15843 virtual) 2018-03-28 00:25:13,074 : INFO : 4 batches submitted to accumulate stats from 256 documents (21113 virtual) 2018-03-28 00:25:13,081 : INFO : 5 batches submitted to accumulate stats from 320 documents (26825 virtual) 2018-03-28 00:25:13,088 : INFO : 6 batches submitted to accumulate stats from 384 documents (32204 virtual) 2018-03-28 00:25:13,205 : INFO : 7 batches submitted to accumulate stats from 448 documents (37741 virtual) 2018-03-28 00:25:13,217 : INFO : 8 batches submitted to accumulate stats from 512 documents (42827 virtual) 2018-03-28 00:25:13,243 : INFO : 9 batches submitted to accumulate stats from 576 documents (48357 virtual) 2018-03-28 00:25:13,357 : INFO : 10 batches submitted to accumulate stats from 640 documents (53652 virtual) 2018-03-28 00:25:13,364 : INFO : 11 batches submitted to accumulate stats from 704 documents (59783 virtual) 2018-03-28 00:25:13,411 : INFO : 12 batches submitted to accumulate stats from 768 documents (64830 virtual) 2018-03-28 00:25:13,508 : INFO : 13 batches submitted to accumulate stats from 832 documents (70885 virtual) 2018-03-28 00:25:13,517 : INFO : 14 batches submitted to accumulate stats from 896 documents (77030 virtual) 2018-03-28 00:25:13,570 : INFO : 15 batches submitted to accumulate stats from 960 documents (82155 virtual) 2018-03-28 00:25:13,658 : INFO : 16 batches submitted to accumulate stats from 1024 documents (87812 virtual) 2018-03-28 00:25:13,665 : INFO : 17 batches submitted to accumulate stats from 1088 documents (93804 virtual) 2018-03-28 00:25:13,700 : INFO : 18 batches submitted to accumulate stats from 1152 documents (99493 virtual) 2018-03-28 00:25:13,821 : INFO : 19 batches submitted to accumulate stats from 1216 documents (104728 virtual) 2018-03-28 00:25:13,827 : INFO : 20 batches submitted to accumulate stats from 1280 documents (109987 virtual) 2018-03-28 00:25:13,845 : INFO : 21 batches submitted to accumulate stats from 1344 documents (115191 virtual) 2018-03-28 00:25:13,970 : INFO : 22 batches submitted to accumulate stats from 1408 documents (120376 virtual) 2018-03-28 00:25:13,982 : INFO : 23 batches submitted to accumulate stats from 1472 documents (126058 virtual) 2018-03-28 00:25:14,011 : INFO : 24 batches submitted to accumulate stats from 1536 documents (131892 virtual) 2018-03-28 00:25:14,113 : INFO : 25 batches submitted to accumulate stats from 1600 documents (137760 virtual) 2018-03-28 00:25:14,120 : INFO : 26 batches submitted to accumulate stats from 1664 documents (143272 virtual) 2018-03-28 00:25:14,158 : INFO : 27 batches submitted to accumulate stats from 1728 documents (148240 virtual) 2018-03-28 00:25:14,259 : INFO : 28 batches submitted to accumulate stats from 1792 documents (153219 virtual) 2018-03-28 00:25:14,290 : INFO : 29 batches submitted to accumulate stats from 1856 documents (158552 virtual) 2018-03-28 00:25:14,311 : INFO : 30 batches submitted to accumulate stats from 1920 documents (163941 virtual) 2018-03-28 00:25:14,393 : INFO : 31 batches submitted to accumulate stats from 1984 documents (169520 virtual) 2018-03-28 00:25:14,443 : INFO : 32 batches submitted to accumulate stats from 2048 documents (175298 virtual) 2018-03-28 00:25:14,488 : INFO : 33 batches submitted to accumulate stats from 2112 documents (181535 virtual) 2018-03-28 00:25:14,521 : INFO : 34 batches submitted to accumulate stats from 2176 documents (186237 virtual) 2018-03-28 00:25:14,584 : INFO : 35 batches submitted to accumulate stats from 2240 documents (191702 virtual) 2018-03-28 00:25:14,656 : INFO : 36 batches submitted to accumulate stats from 2304 documents (197243 virtual) 2018-03-28 00:25:14,663 : INFO : 37 batches submitted to accumulate stats from 2368 documents (203024 virtual) 2018-03-28 00:25:14,733 : INFO : 38 batches submitted to accumulate stats from 2432 documents (208987 virtual) 2018-03-28 00:25:14,800 : INFO : 39 batches submitted to accumulate stats from 2496 documents (215014 virtual) 2018-03-28 00:25:14,804 : INFO : 40 batches submitted to accumulate stats from 2560 documents (220325 virtual) 2018-03-28 00:25:14,893 : INFO : 41 batches submitted to accumulate stats from 2624 documents (225941 virtual) 2018-03-28 00:25:14,940 : INFO : 42 batches submitted to accumulate stats from 2688 documents (231434 virtual) 2018-03-28 00:25:14,955 : INFO : 43 batches submitted to accumulate stats from 2752 documents (237883 virtual) 2018-03-28 00:25:15,060 : INFO : 44 batches submitted to accumulate stats from 2816 documents (243186 virtual) 2018-03-28 00:25:15,092 : INFO : 45 batches submitted to accumulate stats from 2880 documents (248092 virtual) 2018-03-28 00:25:15,105 : INFO : 46 batches submitted to accumulate stats from 2944 documents (253637 virtual) 2018-03-28 00:25:15,194 : INFO : 47 batches submitted to accumulate stats from 3008 documents (259111 virtual) 2018-03-28 00:25:15,236 : INFO : 48 batches submitted to accumulate stats from 3072 documents (264929 virtual) 2018-03-28 00:25:15,272 : INFO : 49 batches submitted to accumulate stats from 3136 documents (270342 virtual) 2018-03-28 00:25:15,345 : INFO : 50 batches submitted to accumulate stats from 3200 documents (275370 virtual) 2018-03-28 00:25:15,397 : INFO : 51 batches submitted to accumulate stats from 3264 documents (281280 virtual) 2018-03-28 00:25:15,414 : INFO : 52 batches submitted to accumulate stats from 3328 documents (286979 virtual) 2018-03-28 00:25:15,513 : INFO : 53 batches submitted to accumulate stats from 3392 documents (292361 virtual) 2018-03-28 00:25:15,531 : INFO : 54 batches submitted to accumulate stats from 3456 documents (298359 virtual) 2018-03-28 00:25:15,569 : INFO : 55 batches submitted to accumulate stats from 3520 documents (304492 virtual) 2018-03-28 00:25:15,645 : INFO : 56 batches submitted to accumulate stats from 3584 documents (308327 virtual) 2018-03-28 00:25:15,845 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:15,851 : INFO : accumulator serialized 2018-03-28 00:25:15,892 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:15,892 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:15,898 : INFO : accumulator serialized 2018-03-28 00:25:15,897 : INFO : accumulator serialized 2018-03-28 00:25:15,933 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:25:15,965 : INFO : accumulated word occurrence stats for 308484 virtual documents 2018-03-28 00:25:16,108 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_15 2018-03-28 00:25:16,110 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_15.expElogbeta.npy with mmap=None 2018-03-28 00:25:16,113 : INFO : setting ignored attribute id2word to None 2018-03-28 00:25:16,113 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:25:16,114 : INFO : setting ignored attribute state to None 2018-03-28 00:25:16,114 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_15 2018-03-28 00:25:16,115 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_15.state 2018-03-28 00:25:16,122 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_15.state 2018-03-28 00:25:16,136 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:25:16,171 : INFO : 1 batches submitted to accumulate stats from 64 documents (5664 virtual) 2018-03-28 00:25:16,179 : INFO : 2 batches submitted to accumulate stats from 128 documents (10533 virtual) 2018-03-28 00:25:16,186 : INFO : 3 batches submitted to accumulate stats from 192 documents (15989 virtual) 2018-03-28 00:25:16,194 : INFO : 4 batches submitted to accumulate stats from 256 documents (21250 virtual) 2018-03-28 00:25:16,202 : INFO : 5 batches submitted to accumulate stats from 320 documents (26908 virtual) 2018-03-28 00:25:16,209 : INFO : 6 batches submitted to accumulate stats from 384 documents (32308 virtual) 2018-03-28 00:25:16,336 : INFO : 7 batches submitted to accumulate stats from 448 documents (37734 virtual) 2018-03-28 00:25:16,343 : INFO : 8 batches submitted to accumulate stats from 512 documents (42666 virtual) 2018-03-28 00:25:16,387 : INFO : 9 batches submitted to accumulate stats from 576 documents (48261 virtual) 2018-03-28 00:25:16,475 : INFO : 10 batches submitted to accumulate stats from 640 documents (53570 virtual) 2018-03-28 00:25:16,524 : INFO : 11 batches submitted to accumulate stats from 704 documents (59720 virtual) 2018-03-28 00:25:16,555 : INFO : 12 batches submitted to accumulate stats from 768 documents (64712 virtual) 2018-03-28 00:25:16,627 : INFO : 13 batches submitted to accumulate stats from 832 documents (70811 virtual) 2018-03-28 00:25:16,676 : INFO : 14 batches submitted to accumulate stats from 896 documents (76931 virtual) 2018-03-28 00:25:16,723 : INFO : 15 batches submitted to accumulate stats from 960 documents (81830 virtual) 2018-03-28 00:25:16,783 : INFO : 16 batches submitted to accumulate stats from 1024 documents (87697 virtual) 2018-03-28 00:25:16,847 : INFO : 17 batches submitted to accumulate stats from 1088 documents (93729 virtual) 2018-03-28 00:25:16,880 : INFO : 18 batches submitted to accumulate stats from 1152 documents (99399 virtual) 2018-03-28 00:25:16,969 : INFO : 19 batches submitted to accumulate stats from 1216 documents (104563 virtual) 2018-03-28 00:25:17,026 : INFO : 20 batches submitted to accumulate stats from 1280 documents (109840 virtual) 2018-03-28 00:25:17,033 : INFO : 21 batches submitted to accumulate stats from 1344 documents (115021 virtual) 2018-03-28 00:25:17,136 : INFO : 22 batches submitted to accumulate stats from 1408 documents (119983 virtual) 2018-03-28 00:25:17,201 : INFO : 23 batches submitted to accumulate stats from 1472 documents (125885 virtual) 2018-03-28 00:25:17,221 : INFO : 24 batches submitted to accumulate stats from 1536 documents (131573 virtual) 2018-03-28 00:25:17,317 : INFO : 25 batches submitted to accumulate stats from 1600 documents (137596 virtual) 2018-03-28 00:25:17,380 : INFO : 26 batches submitted to accumulate stats from 1664 documents (142940 virtual) 2018-03-28 00:25:17,402 : INFO : 27 batches submitted to accumulate stats from 1728 documents (148087 virtual) 2018-03-28 00:25:17,490 : INFO : 28 batches submitted to accumulate stats from 1792 documents (152880 virtual) 2018-03-28 00:25:17,564 : INFO : 29 batches submitted to accumulate stats from 1856 documents (158259 virtual) 2018-03-28 00:25:17,574 : INFO : 30 batches submitted to accumulate stats from 1920 documents (163692 virtual) 2018-03-28 00:25:17,661 : INFO : 31 batches submitted to accumulate stats from 1984 documents (169283 virtual) 2018-03-28 00:25:17,724 : INFO : 32 batches submitted to accumulate stats from 2048 documents (175062 virtual) 2018-03-28 00:25:17,735 : INFO : 33 batches submitted to accumulate stats from 2112 documents (181235 virtual) 2018-03-28 00:25:17,793 : INFO : 34 batches submitted to accumulate stats from 2176 documents (185777 virtual) 2018-03-28 00:25:17,894 : INFO : 35 batches submitted to accumulate stats from 2240 documents (191334 virtual) 2018-03-28 00:25:17,901 : INFO : 36 batches submitted to accumulate stats from 2304 documents (197142 virtual) 2018-03-28 00:25:17,951 : INFO : 37 batches submitted to accumulate stats from 2368 documents (202665 virtual) 2018-03-28 00:25:18,056 : INFO : 38 batches submitted to accumulate stats from 2432 documents (208788 virtual) 2018-03-28 00:25:18,091 : INFO : 39 batches submitted to accumulate stats from 2496 documents (214934 virtual) 2018-03-28 00:25:18,101 : INFO : 40 batches submitted to accumulate stats from 2560 documents (219927 virtual) 2018-03-28 00:25:18,208 : INFO : 41 batches submitted to accumulate stats from 2624 documents (225532 virtual) 2018-03-28 00:25:18,238 : INFO : 42 batches submitted to accumulate stats from 2688 documents (231175 virtual) 2018-03-28 00:25:18,307 : INFO : 43 batches submitted to accumulate stats from 2752 documents (237272 virtual) 2018-03-28 00:25:18,372 : INFO : 44 batches submitted to accumulate stats from 2816 documents (242807 virtual) 2018-03-28 00:25:18,410 : INFO : 45 batches submitted to accumulate stats from 2880 documents (247586 virtual) 2018-03-28 00:25:18,473 : INFO : 46 batches submitted to accumulate stats from 2944 documents (253219 virtual) 2018-03-28 00:25:18,533 : INFO : 47 batches submitted to accumulate stats from 3008 documents (258976 virtual) 2018-03-28 00:25:18,567 : INFO : 48 batches submitted to accumulate stats from 3072 documents (264528 virtual) 2018-03-28 00:25:18,651 : INFO : 49 batches submitted to accumulate stats from 3136 documents (270009 virtual) 2018-03-28 00:25:18,700 : INFO : 50 batches submitted to accumulate stats from 3200 documents (275191 virtual) 2018-03-28 00:25:18,715 : INFO : 51 batches submitted to accumulate stats from 3264 documents (280913 virtual) 2018-03-28 00:25:18,825 : INFO : 52 batches submitted to accumulate stats from 3328 documents (286674 virtual) 2018-03-28 00:25:18,869 : INFO : 53 batches submitted to accumulate stats from 3392 documents (292159 virtual) 2018-03-28 00:25:18,876 : INFO : 54 batches submitted to accumulate stats from 3456 documents (297794 virtual) 2018-03-28 00:25:18,988 : INFO : 55 batches submitted to accumulate stats from 3520 documents (304132 virtual) 2018-03-28 00:25:19,017 : INFO : 56 batches submitted to accumulate stats from 3584 documents (308399 virtual) 2018-03-28 00:25:19,215 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:19,222 : INFO : accumulator serialized 2018-03-28 00:25:19,283 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:19,289 : INFO : accumulator serialized 2018-03-28 00:25:19,319 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:19,325 : INFO : accumulator serialized 2018-03-28 00:25:19,361 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:25:19,405 : INFO : accumulated word occurrence stats for 308567 virtual documents 2018-03-28 00:25:19,610 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_20 2018-03-28 00:25:19,612 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_20.expElogbeta.npy with mmap=None 2018-03-28 00:25:19,614 : INFO : setting ignored attribute id2word to None 2018-03-28 00:25:19,615 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:25:19,616 : INFO : setting ignored attribute state to None 2018-03-28 00:25:19,616 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_20 2018-03-28 00:25:19,617 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_20.state 2018-03-28 00:25:19,624 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_20.state 2018-03-28 00:25:19,635 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:25:19,668 : INFO : 1 batches submitted to accumulate stats from 64 documents (5664 virtual) 2018-03-28 00:25:19,675 : INFO : 2 batches submitted to accumulate stats from 128 documents (10419 virtual) 2018-03-28 00:25:19,682 : INFO : 3 batches submitted to accumulate stats from 192 documents (15719 virtual) 2018-03-28 00:25:19,689 : INFO : 4 batches submitted to accumulate stats from 256 documents (21000 virtual) 2018-03-28 00:25:19,696 : INFO : 5 batches submitted to accumulate stats from 320 documents (26529 virtual) 2018-03-28 00:25:19,703 : INFO : 6 batches submitted to accumulate stats from 384 documents (31837 virtual) 2018-03-28 00:25:19,854 : INFO : 7 batches submitted to accumulate stats from 448 documents (37364 virtual) 2018-03-28 00:25:19,874 : INFO : 8 batches submitted to accumulate stats from 512 documents (42466 virtual) 2018-03-28 00:25:19,897 : INFO : 9 batches submitted to accumulate stats from 576 documents (47807 virtual) 2018-03-28 00:25:20,030 : INFO : 10 batches submitted to accumulate stats from 640 documents (53215 virtual) 2018-03-28 00:25:20,074 : INFO : 11 batches submitted to accumulate stats from 704 documents (59102 virtual) 2018-03-28 00:25:20,080 : INFO : 12 batches submitted to accumulate stats from 768 documents (64353 virtual) 2018-03-28 00:25:20,216 : INFO : 13 batches submitted to accumulate stats from 832 documents (70392 virtual) 2018-03-28 00:25:20,246 : INFO : 14 batches submitted to accumulate stats from 896 documents (76633 virtual) 2018-03-28 00:25:20,283 : INFO : 15 batches submitted to accumulate stats from 960 documents (81603 virtual) 2018-03-28 00:25:20,381 : INFO : 16 batches submitted to accumulate stats from 1024 documents (87442 virtual) 2018-03-28 00:25:20,442 : INFO : 17 batches submitted to accumulate stats from 1088 documents (93180 virtual) 2018-03-28 00:25:20,466 : INFO : 18 batches submitted to accumulate stats from 1152 documents (98888 virtual) 2018-03-28 00:25:20,592 : INFO : 19 batches submitted to accumulate stats from 1216 documents (104392 virtual) 2018-03-28 00:25:20,628 : INFO : 20 batches submitted to accumulate stats from 1280 documents (109480 virtual) 2018-03-28 00:25:20,683 : INFO : 21 batches submitted to accumulate stats from 1344 documents (114883 virtual) 2018-03-28 00:25:20,792 : INFO : 22 batches submitted to accumulate stats from 1408 documents (119642 virtual) 2018-03-28 00:25:20,803 : INFO : 23 batches submitted to accumulate stats from 1472 documents (125571 virtual) 2018-03-28 00:25:20,897 : INFO : 24 batches submitted to accumulate stats from 1536 documents (131152 virtual) 2018-03-28 00:25:20,957 : INFO : 25 batches submitted to accumulate stats from 1600 documents (136924 virtual) 2018-03-28 00:25:20,983 : INFO : 26 batches submitted to accumulate stats from 1664 documents (142635 virtual) 2018-03-28 00:25:21,104 : INFO : 27 batches submitted to accumulate stats from 1728 documents (147591 virtual) 2018-03-28 00:25:21,110 : INFO : 28 batches submitted to accumulate stats from 1792 documents (152512 virtual) 2018-03-28 00:25:21,186 : INFO : 29 batches submitted to accumulate stats from 1856 documents (157743 virtual) 2018-03-28 00:25:21,285 : INFO : 30 batches submitted to accumulate stats from 1920 documents (163278 virtual) 2018-03-28 00:25:21,292 : INFO : 31 batches submitted to accumulate stats from 1984 documents (168786 virtual) 2018-03-28 00:25:21,405 : INFO : 32 batches submitted to accumulate stats from 2048 documents (174396 virtual) 2018-03-28 00:25:21,432 : INFO : 33 batches submitted to accumulate stats from 2112 documents (180502 virtual) 2018-03-28 00:25:21,476 : INFO : 34 batches submitted to accumulate stats from 2176 documents (185335 virtual) 2018-03-28 00:25:21,590 : INFO : 35 batches submitted to accumulate stats from 2240 documents (190704 virtual) 2018-03-28 00:25:21,621 : INFO : 36 batches submitted to accumulate stats from 2304 documents (196116 virtual) 2018-03-28 00:25:21,648 : INFO : 37 batches submitted to accumulate stats from 2368 documents (201616 virtual) 2018-03-28 00:25:21,762 : INFO : 38 batches submitted to accumulate stats from 2432 documents (207826 virtual) 2018-03-28 00:25:21,830 : INFO : 39 batches submitted to accumulate stats from 2496 documents (214002 virtual) 2018-03-28 00:25:21,836 : INFO : 40 batches submitted to accumulate stats from 2560 documents (218918 virtual) 2018-03-28 00:25:21,953 : INFO : 41 batches submitted to accumulate stats from 2624 documents (224350 virtual) 2018-03-28 00:25:22,008 : INFO : 42 batches submitted to accumulate stats from 2688 documents (230017 virtual) 2018-03-28 00:25:22,023 : INFO : 43 batches submitted to accumulate stats from 2752 documents (236012 virtual) 2018-03-28 00:25:22,161 : INFO : 44 batches submitted to accumulate stats from 2816 documents (241860 virtual) 2018-03-28 00:25:22,191 : INFO : 45 batches submitted to accumulate stats from 2880 documents (246509 virtual) 2018-03-28 00:25:22,228 : INFO : 46 batches submitted to accumulate stats from 2944 documents (252155 virtual) 2018-03-28 00:25:22,358 : INFO : 47 batches submitted to accumulate stats from 3008 documents (257643 virtual) 2018-03-28 00:25:22,425 : INFO : 48 batches submitted to accumulate stats from 3072 documents (263189 virtual) 2018-03-28 00:25:22,482 : INFO : 49 batches submitted to accumulate stats from 3136 documents (269010 virtual) 2018-03-28 00:25:22,559 : INFO : 50 batches submitted to accumulate stats from 3200 documents (274291 virtual) 2018-03-28 00:25:22,578 : INFO : 51 batches submitted to accumulate stats from 3264 documents (279765 virtual) 2018-03-28 00:25:22,691 : INFO : 52 batches submitted to accumulate stats from 3328 documents (284909 virtual) 2018-03-28 00:25:22,736 : INFO : 53 batches submitted to accumulate stats from 3392 documents (290712 virtual) 2018-03-28 00:25:22,781 : INFO : 54 batches submitted to accumulate stats from 3456 documents (296484 virtual) 2018-03-28 00:25:22,878 : INFO : 55 batches submitted to accumulate stats from 3520 documents (302671 virtual) 2018-03-28 00:25:22,929 : INFO : 56 batches submitted to accumulate stats from 3584 documents (307982 virtual) 2018-03-28 00:25:22,971 : INFO : 57 batches submitted to accumulate stats from 3648 documents (308374 virtual) 2018-03-28 00:25:23,203 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:23,221 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:23,209 : INFO : accumulator serialized 2018-03-28 00:25:23,226 : INFO : accumulator serialized 2018-03-28 00:25:23,299 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:23,304 : INFO : accumulator serialized 2018-03-28 00:25:23,353 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:25:23,438 : INFO : accumulated word occurrence stats for 308556 virtual documents 2018-03-28 00:25:23,706 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_25 2018-03-28 00:25:23,707 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-28 00:25:23,710 : INFO : setting ignored attribute id2word to None 2018-03-28 00:25:23,711 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:25:23,711 : INFO : setting ignored attribute state to None 2018-03-28 00:25:23,713 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_25 2018-03-28 00:25:23,713 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_25.state 2018-03-28 00:25:23,722 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_25.state 2018-03-28 00:25:23,735 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:25:23,770 : INFO : 1 batches submitted to accumulate stats from 64 documents (5664 virtual) 2018-03-28 00:25:23,778 : INFO : 2 batches submitted to accumulate stats from 128 documents (10419 virtual) 2018-03-28 00:25:23,785 : INFO : 3 batches submitted to accumulate stats from 192 documents (15719 virtual) 2018-03-28 00:25:23,792 : INFO : 4 batches submitted to accumulate stats from 256 documents (21000 virtual) 2018-03-28 00:25:23,800 : INFO : 5 batches submitted to accumulate stats from 320 documents (26636 virtual) 2018-03-28 00:25:23,807 : INFO : 6 batches submitted to accumulate stats from 384 documents (31832 virtual) 2018-03-28 00:25:23,970 : INFO : 7 batches submitted to accumulate stats from 448 documents (37352 virtual) 2018-03-28 00:25:24,006 : INFO : 8 batches submitted to accumulate stats from 512 documents (42377 virtual) 2018-03-28 00:25:24,012 : INFO : 9 batches submitted to accumulate stats from 576 documents (47646 virtual) 2018-03-28 00:25:24,159 : INFO : 10 batches submitted to accumulate stats from 640 documents (53138 virtual) 2018-03-28 00:25:24,207 : INFO : 11 batches submitted to accumulate stats from 704 documents (59040 virtual) 2018-03-28 00:25:24,228 : INFO : 12 batches submitted to accumulate stats from 768 documents (64343 virtual) 2018-03-28 00:25:24,333 : INFO : 13 batches submitted to accumulate stats from 832 documents (70303 virtual) 2018-03-28 00:25:24,410 : INFO : 14 batches submitted to accumulate stats from 896 documents (76515 virtual) 2018-03-28 00:25:24,427 : INFO : 15 batches submitted to accumulate stats from 960 documents (81501 virtual) 2018-03-28 00:25:24,513 : INFO : 16 batches submitted to accumulate stats from 1024 documents (87326 virtual) 2018-03-28 00:25:24,592 : INFO : 17 batches submitted to accumulate stats from 1088 documents (93024 virtual) 2018-03-28 00:25:24,630 : INFO : 18 batches submitted to accumulate stats from 1152 documents (98744 virtual) 2018-03-28 00:25:24,699 : INFO : 19 batches submitted to accumulate stats from 1216 documents (104348 virtual) 2018-03-28 00:25:24,816 : INFO : 20 batches submitted to accumulate stats from 1280 documents (109371 virtual) 2018-03-28 00:25:24,857 : INFO : 21 batches submitted to accumulate stats from 1344 documents (114824 virtual) 2018-03-28 00:25:24,876 : INFO : 22 batches submitted to accumulate stats from 1408 documents (119527 virtual) 2018-03-28 00:25:25,017 : INFO : 23 batches submitted to accumulate stats from 1472 documents (125432 virtual) 2018-03-28 00:25:25,076 : INFO : 24 batches submitted to accumulate stats from 1536 documents (131075 virtual) 2018-03-28 00:25:25,090 : INFO : 25 batches submitted to accumulate stats from 1600 documents (136911 virtual) 2018-03-28 00:25:25,183 : INFO : 26 batches submitted to accumulate stats from 1664 documents (142471 virtual) 2018-03-28 00:25:25,255 : INFO : 27 batches submitted to accumulate stats from 1728 documents (147539 virtual) 2018-03-28 00:25:25,278 : INFO : 28 batches submitted to accumulate stats from 1792 documents (152468 virtual) 2018-03-28 00:25:25,388 : INFO : 29 batches submitted to accumulate stats from 1856 documents (157506 virtual) 2018-03-28 00:25:25,467 : INFO : 30 batches submitted to accumulate stats from 1920 documents (163157 virtual) 2018-03-28 00:25:25,494 : INFO : 31 batches submitted to accumulate stats from 1984 documents (168651 virtual) 2018-03-28 00:25:25,578 : INFO : 32 batches submitted to accumulate stats from 2048 documents (174259 virtual) 2018-03-28 00:25:25,655 : INFO : 33 batches submitted to accumulate stats from 2112 documents (180413 virtual) 2018-03-28 00:25:25,677 : INFO : 34 batches submitted to accumulate stats from 2176 documents (185264 virtual) 2018-03-28 00:25:25,756 : INFO : 35 batches submitted to accumulate stats from 2240 documents (190566 virtual) 2018-03-28 00:25:25,858 : INFO : 36 batches submitted to accumulate stats from 2304 documents (196007 virtual) 2018-03-28 00:25:25,870 : INFO : 37 batches submitted to accumulate stats from 2368 documents (201519 virtual) 2018-03-28 00:25:25,972 : INFO : 38 batches submitted to accumulate stats from 2432 documents (207602 virtual) 2018-03-28 00:25:26,026 : INFO : 39 batches submitted to accumulate stats from 2496 documents (213857 virtual) 2018-03-28 00:25:26,063 : INFO : 40 batches submitted to accumulate stats from 2560 documents (218799 virtual) 2018-03-28 00:25:26,188 : INFO : 41 batches submitted to accumulate stats from 2624 documents (224227 virtual) 2018-03-28 00:25:26,235 : INFO : 42 batches submitted to accumulate stats from 2688 documents (229886 virtual) 2018-03-28 00:25:26,245 : INFO : 43 batches submitted to accumulate stats from 2752 documents (235780 virtual) 2018-03-28 00:25:26,396 : INFO : 44 batches submitted to accumulate stats from 2816 documents (241707 virtual) 2018-03-28 00:25:26,409 : INFO : 45 batches submitted to accumulate stats from 2880 documents (246368 virtual) 2018-03-28 00:25:26,470 : INFO : 46 batches submitted to accumulate stats from 2944 documents (251922 virtual) 2018-03-28 00:25:26,601 : INFO : 47 batches submitted to accumulate stats from 3008 documents (257323 virtual) 2018-03-28 00:25:26,608 : INFO : 48 batches submitted to accumulate stats from 3072 documents (262872 virtual) 2018-03-28 00:25:26,657 : INFO : 49 batches submitted to accumulate stats from 3136 documents (268678 virtual) 2018-03-28 00:25:26,770 : INFO : 50 batches submitted to accumulate stats from 3200 documents (274036 virtual) 2018-03-28 00:25:26,834 : INFO : 51 batches submitted to accumulate stats from 3264 documents (279299 virtual) 2018-03-28 00:25:26,847 : INFO : 52 batches submitted to accumulate stats from 3328 documents (284511 virtual) 2018-03-28 00:25:26,978 : INFO : 53 batches submitted to accumulate stats from 3392 documents (290305 virtual) 2018-03-28 00:25:27,027 : INFO : 54 batches submitted to accumulate stats from 3456 documents (296012 virtual) 2018-03-28 00:25:27,055 : INFO : 55 batches submitted to accumulate stats from 3520 documents (302320 virtual) 2018-03-28 00:25:27,166 : INFO : 56 batches submitted to accumulate stats from 3584 documents (307574 virtual) 2018-03-28 00:25:27,218 : INFO : 57 batches submitted to accumulate stats from 3648 documents (308458 virtual) 2018-03-28 00:25:27,494 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:27,519 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:27,549 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:27,500 : INFO : accumulator serialized 2018-03-28 00:25:27,524 : INFO : accumulator serialized 2018-03-28 00:25:27,555 : INFO : accumulator serialized 2018-03-28 00:25:27,630 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:25:27,737 : INFO : accumulated word occurrence stats for 308656 virtual documents 2018-03-28 00:25:28,075 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_30 2018-03-28 00:25:28,077 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-28 00:25:28,079 : INFO : setting ignored attribute id2word to None 2018-03-28 00:25:28,080 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:25:28,081 : INFO : setting ignored attribute state to None 2018-03-28 00:25:28,081 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_30 2018-03-28 00:25:28,082 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_30.state 2018-03-28 00:25:28,091 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_30.state 2018-03-28 00:25:28,106 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:25:28,140 : INFO : 1 batches submitted to accumulate stats from 64 documents (5664 virtual) 2018-03-28 00:25:28,148 : INFO : 2 batches submitted to accumulate stats from 128 documents (10419 virtual) 2018-03-28 00:25:28,156 : INFO : 3 batches submitted to accumulate stats from 192 documents (15719 virtual) 2018-03-28 00:25:28,163 : INFO : 4 batches submitted to accumulate stats from 256 documents (21000 virtual) 2018-03-28 00:25:28,171 : INFO : 5 batches submitted to accumulate stats from 320 documents (26529 virtual) 2018-03-28 00:25:28,178 : INFO : 6 batches submitted to accumulate stats from 384 documents (31703 virtual) 2018-03-28 00:25:28,340 : INFO : 7 batches submitted to accumulate stats from 448 documents (37361 virtual) 2018-03-28 00:25:28,376 : INFO : 8 batches submitted to accumulate stats from 512 documents (42386 virtual) 2018-03-28 00:25:28,396 : INFO : 9 batches submitted to accumulate stats from 576 documents (47655 virtual) 2018-03-28 00:25:28,567 : INFO : 10 batches submitted to accumulate stats from 640 documents (53147 virtual) 2018-03-28 00:25:28,575 : INFO : 11 batches submitted to accumulate stats from 704 documents (59049 virtual) 2018-03-28 00:25:28,601 : INFO : 12 batches submitted to accumulate stats from 768 documents (64352 virtual) 2018-03-28 00:25:28,777 : INFO : 13 batches submitted to accumulate stats from 832 documents (70312 virtual) 2018-03-28 00:25:28,787 : INFO : 14 batches submitted to accumulate stats from 896 documents (76524 virtual) 2018-03-28 00:25:28,807 : INFO : 15 batches submitted to accumulate stats from 960 documents (81510 virtual) 2018-03-28 00:25:28,982 : INFO : 16 batches submitted to accumulate stats from 1024 documents (87335 virtual) 2018-03-28 00:25:28,990 : INFO : 17 batches submitted to accumulate stats from 1088 documents (93033 virtual) 2018-03-28 00:25:29,028 : INFO : 18 batches submitted to accumulate stats from 1152 documents (98753 virtual) 2018-03-28 00:25:29,199 : INFO : 19 batches submitted to accumulate stats from 1216 documents (104357 virtual) 2018-03-28 00:25:29,233 : INFO : 20 batches submitted to accumulate stats from 1280 documents (109380 virtual) 2018-03-28 00:25:29,240 : INFO : 21 batches submitted to accumulate stats from 1344 documents (114833 virtual) 2018-03-28 00:25:29,416 : INFO : 22 batches submitted to accumulate stats from 1408 documents (119536 virtual) 2018-03-28 00:25:29,449 : INFO : 23 batches submitted to accumulate stats from 1472 documents (125441 virtual) 2018-03-28 00:25:29,456 : INFO : 24 batches submitted to accumulate stats from 1536 documents (131084 virtual) 2018-03-28 00:25:29,616 : INFO : 25 batches submitted to accumulate stats from 1600 documents (136920 virtual) 2018-03-28 00:25:29,659 : INFO : 26 batches submitted to accumulate stats from 1664 documents (142480 virtual) 2018-03-28 00:25:29,675 : INFO : 27 batches submitted to accumulate stats from 1728 documents (147548 virtual) 2018-03-28 00:25:29,789 : INFO : 28 batches submitted to accumulate stats from 1792 documents (152477 virtual) 2018-03-28 00:25:29,860 : INFO : 29 batches submitted to accumulate stats from 1856 documents (157515 virtual) 2018-03-28 00:25:29,924 : INFO : 30 batches submitted to accumulate stats from 1920 documents (163023 virtual) 2018-03-28 00:25:30,030 : INFO : 31 batches submitted to accumulate stats from 1984 documents (168434 virtual) 2018-03-28 00:25:30,071 : INFO : 32 batches submitted to accumulate stats from 2048 documents (174247 virtual) 2018-03-28 00:25:30,120 : INFO : 33 batches submitted to accumulate stats from 2112 documents (180258 virtual) 2018-03-28 00:25:30,224 : INFO : 34 batches submitted to accumulate stats from 2176 documents (185262 virtual) 2018-03-28 00:25:30,277 : INFO : 35 batches submitted to accumulate stats from 2240 documents (190501 virtual) 2018-03-28 00:25:30,326 : INFO : 36 batches submitted to accumulate stats from 2304 documents (195924 virtual) 2018-03-28 00:25:30,421 : INFO : 37 batches submitted to accumulate stats from 2368 documents (201422 virtual) 2018-03-28 00:25:30,485 : INFO : 38 batches submitted to accumulate stats from 2432 documents (207564 virtual) 2018-03-28 00:25:30,571 : INFO : 39 batches submitted to accumulate stats from 2496 documents (213811 virtual) 2018-03-28 00:25:30,626 : INFO : 40 batches submitted to accumulate stats from 2560 documents (218707 virtual) 2018-03-28 00:25:30,689 : INFO : 41 batches submitted to accumulate stats from 2624 documents (224207 virtual) 2018-03-28 00:25:30,793 : INFO : 42 batches submitted to accumulate stats from 2688 documents (229874 virtual) 2018-03-28 00:25:30,856 : INFO : 43 batches submitted to accumulate stats from 2752 documents (235789 virtual) 2018-03-28 00:25:30,897 : INFO : 44 batches submitted to accumulate stats from 2816 documents (241716 virtual) 2018-03-28 00:25:31,041 : INFO : 45 batches submitted to accumulate stats from 2880 documents (246377 virtual) 2018-03-28 00:25:31,048 : INFO : 46 batches submitted to accumulate stats from 2944 documents (251931 virtual) 2018-03-28 00:25:31,097 : INFO : 47 batches submitted to accumulate stats from 3008 documents (257332 virtual) 2018-03-28 00:25:31,270 : INFO : 48 batches submitted to accumulate stats from 3072 documents (262813 virtual) 2018-03-28 00:25:31,277 : INFO : 49 batches submitted to accumulate stats from 3136 documents (268666 virtual) 2018-03-28 00:25:31,294 : INFO : 50 batches submitted to accumulate stats from 3200 documents (274015 virtual) 2018-03-28 00:25:31,461 : INFO : 51 batches submitted to accumulate stats from 3264 documents (279231 virtual) 2018-03-28 00:25:31,468 : INFO : 52 batches submitted to accumulate stats from 3328 documents (284428 virtual) 2018-03-28 00:25:31,521 : INFO : 53 batches submitted to accumulate stats from 3392 documents (290241 virtual) 2018-03-28 00:25:31,680 : INFO : 54 batches submitted to accumulate stats from 3456 documents (295925 virtual) 2018-03-28 00:25:31,700 : INFO : 55 batches submitted to accumulate stats from 3520 documents (302155 virtual) 2018-03-28 00:25:31,730 : INFO : 56 batches submitted to accumulate stats from 3584 documents (307589 virtual) 2018-03-28 00:25:31,870 : INFO : 57 batches submitted to accumulate stats from 3648 documents (308473 virtual) 2018-03-28 00:25:32,103 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:32,139 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:32,154 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:32,109 : INFO : accumulator serialized 2018-03-28 00:25:32,145 : INFO : accumulator serialized 2018-03-28 00:25:32,161 : INFO : accumulator serialized 2018-03-28 00:25:32,270 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:25:32,410 : INFO : accumulated word occurrence stats for 308662 virtual documents 2018-03-28 00:25:32,809 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_35 2018-03-28 00:25:32,811 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_35.expElogbeta.npy with mmap=None 2018-03-28 00:25:32,814 : INFO : setting ignored attribute id2word to None 2018-03-28 00:25:32,815 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:25:32,815 : INFO : setting ignored attribute state to None 2018-03-28 00:25:32,816 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_35 2018-03-28 00:25:32,817 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_35.state 2018-03-28 00:25:32,828 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_35.state 2018-03-28 00:25:32,845 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:25:32,881 : INFO : 1 batches submitted to accumulate stats from 64 documents (5664 virtual) 2018-03-28 00:25:32,888 : INFO : 2 batches submitted to accumulate stats from 128 documents (10419 virtual) 2018-03-28 00:25:32,895 : INFO : 3 batches submitted to accumulate stats from 192 documents (15719 virtual) 2018-03-28 00:25:32,902 : INFO : 4 batches submitted to accumulate stats from 256 documents (21000 virtual) 2018-03-28 00:25:32,910 : INFO : 5 batches submitted to accumulate stats from 320 documents (26464 virtual) 2018-03-28 00:25:32,917 : INFO : 6 batches submitted to accumulate stats from 384 documents (31708 virtual) 2018-03-28 00:25:33,101 : INFO : 7 batches submitted to accumulate stats from 448 documents (37371 virtual) 2018-03-28 00:25:33,122 : INFO : 8 batches submitted to accumulate stats from 512 documents (42473 virtual) 2018-03-28 00:25:33,129 : INFO : 9 batches submitted to accumulate stats from 576 documents (47814 virtual) 2018-03-28 00:25:33,314 : INFO : 10 batches submitted to accumulate stats from 640 documents (53222 virtual) 2018-03-28 00:25:33,343 : INFO : 11 batches submitted to accumulate stats from 704 documents (59109 virtual) 2018-03-28 00:25:33,350 : INFO : 12 batches submitted to accumulate stats from 768 documents (64360 virtual) 2018-03-28 00:25:33,524 : INFO : 13 batches submitted to accumulate stats from 832 documents (70399 virtual) 2018-03-28 00:25:33,535 : INFO : 14 batches submitted to accumulate stats from 896 documents (76398 virtual) 2018-03-28 00:25:33,588 : INFO : 15 batches submitted to accumulate stats from 960 documents (81446 virtual) 2018-03-28 00:25:33,750 : INFO : 16 batches submitted to accumulate stats from 1024 documents (87191 virtual) 2018-03-28 00:25:33,763 : INFO : 17 batches submitted to accumulate stats from 1088 documents (92872 virtual) 2018-03-28 00:25:33,787 : INFO : 18 batches submitted to accumulate stats from 1152 documents (98568 virtual) 2018-03-28 00:25:33,963 : INFO : 19 batches submitted to accumulate stats from 1216 documents (104059 virtual) 2018-03-28 00:25:33,991 : INFO : 20 batches submitted to accumulate stats from 1280 documents (109182 virtual) 2018-03-28 00:25:34,011 : INFO : 21 batches submitted to accumulate stats from 1344 documents (114494 virtual) 2018-03-28 00:25:34,191 : INFO : 22 batches submitted to accumulate stats from 1408 documents (119431 virtual) 2018-03-28 00:25:34,218 : INFO : 23 batches submitted to accumulate stats from 1472 documents (125124 virtual) 2018-03-28 00:25:34,229 : INFO : 24 batches submitted to accumulate stats from 1536 documents (130914 virtual) 2018-03-28 00:25:34,396 : INFO : 25 batches submitted to accumulate stats from 1600 documents (136736 virtual) 2018-03-28 00:25:34,427 : INFO : 26 batches submitted to accumulate stats from 1664 documents (142384 virtual) 2018-03-28 00:25:34,459 : INFO : 27 batches submitted to accumulate stats from 1728 documents (147311 virtual) 2018-03-28 00:25:34,609 : INFO : 28 batches submitted to accumulate stats from 1792 documents (152294 virtual) 2018-03-28 00:25:34,616 : INFO : 29 batches submitted to accumulate stats from 1856 documents (157255 virtual) 2018-03-28 00:25:34,714 : INFO : 30 batches submitted to accumulate stats from 1920 documents (162594 virtual) 2018-03-28 00:25:34,807 : INFO : 31 batches submitted to accumulate stats from 1984 documents (168004 virtual) 2018-03-28 00:25:34,857 : INFO : 32 batches submitted to accumulate stats from 2048 documents (173909 virtual) 2018-03-28 00:25:34,924 : INFO : 33 batches submitted to accumulate stats from 2112 documents (179932 virtual) 2018-03-28 00:25:35,020 : INFO : 34 batches submitted to accumulate stats from 2176 documents (185090 virtual) 2018-03-28 00:25:35,063 : INFO : 35 batches submitted to accumulate stats from 2240 documents (190308 virtual) 2018-03-28 00:25:35,120 : INFO : 36 batches submitted to accumulate stats from 2304 documents (195522 virtual) 2018-03-28 00:25:35,243 : INFO : 37 batches submitted to accumulate stats from 2368 documents (201004 virtual) 2018-03-28 00:25:35,289 : INFO : 38 batches submitted to accumulate stats from 2432 documents (207087 virtual) 2018-03-28 00:25:35,347 : INFO : 39 batches submitted to accumulate stats from 2496 documents (213279 virtual) 2018-03-28 00:25:35,459 : INFO : 40 batches submitted to accumulate stats from 2560 documents (218336 virtual) 2018-03-28 00:25:35,512 : INFO : 41 batches submitted to accumulate stats from 2624 documents (223652 virtual) 2018-03-28 00:25:35,555 : INFO : 42 batches submitted to accumulate stats from 2688 documents (229365 virtual) 2018-03-28 00:25:35,680 : INFO : 43 batches submitted to accumulate stats from 2752 documents (235217 virtual) 2018-03-28 00:25:35,713 : INFO : 44 batches submitted to accumulate stats from 2816 documents (241275 virtual) 2018-03-28 00:25:35,815 : INFO : 45 batches submitted to accumulate stats from 2880 documents (246155 virtual) 2018-03-28 00:25:35,885 : INFO : 46 batches submitted to accumulate stats from 2944 documents (251513 virtual) 2018-03-28 00:25:35,910 : INFO : 47 batches submitted to accumulate stats from 3008 documents (256728 virtual) 2018-03-28 00:25:36,062 : INFO : 48 batches submitted to accumulate stats from 3072 documents (262160 virtual) 2018-03-28 00:25:36,090 : INFO : 49 batches submitted to accumulate stats from 3136 documents (268217 virtual) 2018-03-28 00:25:36,174 : INFO : 50 batches submitted to accumulate stats from 3200 documents (273593 virtual) 2018-03-28 00:25:36,256 : INFO : 51 batches submitted to accumulate stats from 3264 documents (278537 virtual) 2018-03-28 00:25:36,294 : INFO : 52 batches submitted to accumulate stats from 3328 documents (283906 virtual) 2018-03-28 00:25:36,369 : INFO : 53 batches submitted to accumulate stats from 3392 documents (290031 virtual) 2018-03-28 00:25:36,461 : INFO : 54 batches submitted to accumulate stats from 3456 documents (295440 virtual) 2018-03-28 00:25:36,564 : INFO : 55 batches submitted to accumulate stats from 3520 documents (301390 virtual) 2018-03-28 00:25:36,595 : INFO : 56 batches submitted to accumulate stats from 3584 documents (306974 virtual) 2018-03-28 00:25:36,654 : INFO : 57 batches submitted to accumulate stats from 3648 documents (308674 virtual) 2018-03-28 00:25:36,941 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:36,999 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:36,946 : INFO : accumulator serialized 2018-03-28 00:25:37,054 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:37,005 : INFO : accumulator serialized 2018-03-28 00:25:37,060 : INFO : accumulator serialized 2018-03-28 00:25:37,155 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:25:37,305 : INFO : accumulated word occurrence stats for 308859 virtual documents 2018-03-28 00:25:37,744 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_40 2018-03-28 00:25:37,746 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-28 00:25:37,749 : INFO : setting ignored attribute id2word to None 2018-03-28 00:25:37,749 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:25:37,750 : INFO : setting ignored attribute state to None 2018-03-28 00:25:37,750 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_40 2018-03-28 00:25:37,751 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_40.state 2018-03-28 00:25:37,763 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_40.state 2018-03-28 00:25:37,782 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:25:37,819 : INFO : 1 batches submitted to accumulate stats from 64 documents (5664 virtual) 2018-03-28 00:25:37,827 : INFO : 2 batches submitted to accumulate stats from 128 documents (10419 virtual) 2018-03-28 00:25:37,835 : INFO : 3 batches submitted to accumulate stats from 192 documents (15719 virtual) 2018-03-28 00:25:37,843 : INFO : 4 batches submitted to accumulate stats from 256 documents (21000 virtual) 2018-03-28 00:25:37,851 : INFO : 5 batches submitted to accumulate stats from 320 documents (26464 virtual) 2018-03-28 00:25:37,858 : INFO : 6 batches submitted to accumulate stats from 384 documents (31629 virtual) 2018-03-28 00:25:38,031 : INFO : 7 batches submitted to accumulate stats from 448 documents (37187 virtual) 2018-03-28 00:25:38,086 : INFO : 8 batches submitted to accumulate stats from 512 documents (42347 virtual) 2018-03-28 00:25:38,095 : INFO : 9 batches submitted to accumulate stats from 576 documents (47579 virtual) 2018-03-28 00:25:38,266 : INFO : 10 batches submitted to accumulate stats from 640 documents (53119 virtual) 2018-03-28 00:25:38,303 : INFO : 11 batches submitted to accumulate stats from 704 documents (59014 virtual) 2018-03-28 00:25:38,366 : INFO : 12 batches submitted to accumulate stats from 768 documents (64333 virtual) 2018-03-28 00:25:38,489 : INFO : 13 batches submitted to accumulate stats from 832 documents (70217 virtual) 2018-03-28 00:25:38,538 : INFO : 14 batches submitted to accumulate stats from 896 documents (76278 virtual) 2018-03-28 00:25:38,592 : INFO : 15 batches submitted to accumulate stats from 960 documents (81280 virtual) 2018-03-28 00:25:38,753 : INFO : 16 batches submitted to accumulate stats from 1024 documents (86955 virtual) 2018-03-28 00:25:38,764 : INFO : 17 batches submitted to accumulate stats from 1088 documents (92598 virtual) 2018-03-28 00:25:38,808 : INFO : 18 batches submitted to accumulate stats from 1152 documents (98386 virtual) 2018-03-28 00:25:38,996 : INFO : 19 batches submitted to accumulate stats from 1216 documents (103916 virtual) 2018-03-28 00:25:39,010 : INFO : 20 batches submitted to accumulate stats from 1280 documents (108948 virtual) 2018-03-28 00:25:39,027 : INFO : 21 batches submitted to accumulate stats from 1344 documents (114259 virtual) 2018-03-28 00:25:39,211 : INFO : 22 batches submitted to accumulate stats from 1408 documents (119338 virtual) 2018-03-28 00:25:39,267 : INFO : 23 batches submitted to accumulate stats from 1472 documents (124909 virtual) 2018-03-28 00:25:39,281 : INFO : 24 batches submitted to accumulate stats from 1536 documents (130686 virtual) 2018-03-28 00:25:39,446 : INFO : 25 batches submitted to accumulate stats from 1600 documents (136555 virtual) 2018-03-28 00:25:39,491 : INFO : 26 batches submitted to accumulate stats from 1664 documents (142139 virtual) 2018-03-28 00:25:39,511 : INFO : 27 batches submitted to accumulate stats from 1728 documents (147021 virtual) 2018-03-28 00:25:39,680 : INFO : 28 batches submitted to accumulate stats from 1792 documents (152035 virtual) 2018-03-28 00:25:39,717 : INFO : 29 batches submitted to accumulate stats from 1856 documents (157004 virtual) 2018-03-28 00:25:39,737 : INFO : 30 batches submitted to accumulate stats from 1920 documents (162463 virtual) 2018-03-28 00:25:39,928 : INFO : 31 batches submitted to accumulate stats from 1984 documents (167798 virtual) 2018-03-28 00:25:39,943 : INFO : 32 batches submitted to accumulate stats from 2048 documents (173647 virtual) 2018-03-28 00:25:39,968 : INFO : 33 batches submitted to accumulate stats from 2112 documents (179632 virtual) 2018-03-28 00:25:40,145 : INFO : 34 batches submitted to accumulate stats from 2176 documents (184747 virtual) 2018-03-28 00:25:40,152 : INFO : 35 batches submitted to accumulate stats from 2240 documents (189960 virtual) 2018-03-28 00:25:40,207 : INFO : 36 batches submitted to accumulate stats from 2304 documents (195310 virtual) 2018-03-28 00:25:40,374 : INFO : 37 batches submitted to accumulate stats from 2368 documents (200572 virtual) 2018-03-28 00:25:40,384 : INFO : 38 batches submitted to accumulate stats from 2432 documents (206613 virtual) 2018-03-28 00:25:40,456 : INFO : 39 batches submitted to accumulate stats from 2496 documents (212418 virtual) 2018-03-28 00:25:40,574 : INFO : 40 batches submitted to accumulate stats from 2560 documents (217855 virtual) 2018-03-28 00:25:40,609 : INFO : 41 batches submitted to accumulate stats from 2624 documents (223217 virtual) 2018-03-28 00:25:40,693 : INFO : 42 batches submitted to accumulate stats from 2688 documents (229087 virtual) 2018-03-28 00:25:40,800 : INFO : 43 batches submitted to accumulate stats from 2752 documents (234493 virtual) 2018-03-28 00:25:40,858 : INFO : 44 batches submitted to accumulate stats from 2816 documents (240703 virtual) 2018-03-28 00:25:40,922 : INFO : 45 batches submitted to accumulate stats from 2880 documents (245736 virtual) 2018-03-28 00:25:41,057 : INFO : 46 batches submitted to accumulate stats from 2944 documents (250934 virtual) 2018-03-28 00:25:41,082 : INFO : 47 batches submitted to accumulate stats from 3008 documents (256363 virtual) 2018-03-28 00:25:41,163 : INFO : 48 batches submitted to accumulate stats from 3072 documents (261469 virtual) 2018-03-28 00:25:41,279 : INFO : 49 batches submitted to accumulate stats from 3136 documents (267727 virtual) 2018-03-28 00:25:41,318 : INFO : 50 batches submitted to accumulate stats from 3200 documents (272886 virtual) 2018-03-28 00:25:41,385 : INFO : 51 batches submitted to accumulate stats from 3264 documents (277857 virtual) 2018-03-28 00:25:41,501 : INFO : 52 batches submitted to accumulate stats from 3328 documents (283361 virtual) 2018-03-28 00:25:41,554 : INFO : 53 batches submitted to accumulate stats from 3392 documents (289483 virtual) 2018-03-28 00:25:41,604 : INFO : 54 batches submitted to accumulate stats from 3456 documents (294839 virtual) 2018-03-28 00:25:41,773 : INFO : 55 batches submitted to accumulate stats from 3520 documents (300772 virtual) 2018-03-28 00:25:41,787 : INFO : 56 batches submitted to accumulate stats from 3584 documents (306550 virtual) 2018-03-28 00:25:41,803 : INFO : 57 batches submitted to accumulate stats from 3648 documents (308663 virtual) 2018-03-28 00:25:42,161 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:42,233 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:42,168 : INFO : accumulator serialized 2018-03-28 00:25:42,255 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:42,262 : INFO : accumulator serialized 2018-03-28 00:25:42,239 : INFO : accumulator serialized 2018-03-28 00:25:42,400 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:25:42,585 : INFO : accumulated word occurrence stats for 308874 virtual documents 2018-03-28 00:25:43,068 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_50 2018-03-28 00:25:43,070 : INFO : loading expElogbeta from ../src/tmrest_meta//ldamodel_nb_topics_50.expElogbeta.npy with mmap=None 2018-03-28 00:25:43,074 : INFO : setting ignored attribute id2word to None 2018-03-28 00:25:43,074 : INFO : setting ignored attribute dispatcher to None 2018-03-28 00:25:43,075 : INFO : setting ignored attribute state to None 2018-03-28 00:25:43,075 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_50 2018-03-28 00:25:43,076 : INFO : loading LdaModel object from ../src/tmrest_meta//ldamodel_nb_topics_50.state 2018-03-28 00:25:43,091 : INFO : loaded ../src/tmrest_meta//ldamodel_nb_topics_50.state 2018-03-28 00:25:43,114 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-28 00:25:43,151 : INFO : 1 batches submitted to accumulate stats from 64 documents (5664 virtual) 2018-03-28 00:25:43,159 : INFO : 2 batches submitted to accumulate stats from 128 documents (10419 virtual) 2018-03-28 00:25:43,166 : INFO : 3 batches submitted to accumulate stats from 192 documents (15719 virtual) 2018-03-28 00:25:43,172 : INFO : 4 batches submitted to accumulate stats from 256 documents (21000 virtual) 2018-03-28 00:25:43,181 : INFO : 5 batches submitted to accumulate stats from 320 documents (26529 virtual) 2018-03-28 00:25:43,188 : INFO : 6 batches submitted to accumulate stats from 384 documents (31703 virtual) 2018-03-28 00:25:43,409 : INFO : 7 batches submitted to accumulate stats from 448 documents (37180 virtual) 2018-03-28 00:25:43,424 : INFO : 8 batches submitted to accumulate stats from 512 documents (42303 virtual) 2018-03-28 00:25:43,436 : INFO : 9 batches submitted to accumulate stats from 576 documents (47476 virtual) 2018-03-28 00:25:43,648 : INFO : 10 batches submitted to accumulate stats from 640 documents (52956 virtual) 2018-03-28 00:25:43,688 : INFO : 11 batches submitted to accumulate stats from 704 documents (58907 virtual) 2018-03-28 00:25:43,711 : INFO : 12 batches submitted to accumulate stats from 768 documents (64320 virtual) 2018-03-28 00:25:43,871 : INFO : 13 batches submitted to accumulate stats from 832 documents (70077 virtual) 2018-03-28 00:25:43,925 : INFO : 14 batches submitted to accumulate stats from 896 documents (76170 virtual) 2018-03-28 00:25:43,976 : INFO : 15 batches submitted to accumulate stats from 960 documents (81264 virtual) 2018-03-28 00:25:44,105 : INFO : 16 batches submitted to accumulate stats from 1024 documents (86854 virtual) 2018-03-28 00:25:44,173 : INFO : 17 batches submitted to accumulate stats from 1088 documents (92560 virtual) 2018-03-28 00:25:44,275 : INFO : 18 batches submitted to accumulate stats from 1152 documents (98362 virtual) 2018-03-28 00:25:44,382 : INFO : 19 batches submitted to accumulate stats from 1216 documents (103856 virtual) 2018-03-28 00:25:44,439 : INFO : 20 batches submitted to accumulate stats from 1280 documents (108843 virtual) 2018-03-28 00:25:44,498 : INFO : 21 batches submitted to accumulate stats from 1344 documents (114128 virtual) 2018-03-28 00:25:44,643 : INFO : 22 batches submitted to accumulate stats from 1408 documents (119207 virtual) 2018-03-28 00:25:44,698 : INFO : 23 batches submitted to accumulate stats from 1472 documents (124710 virtual) 2018-03-28 00:25:44,795 : INFO : 24 batches submitted to accumulate stats from 1536 documents (130521 virtual) 2018-03-28 00:25:44,957 : INFO : 25 batches submitted to accumulate stats from 1600 documents (136395 virtual) 2018-03-28 00:25:44,972 : INFO : 26 batches submitted to accumulate stats from 1664 documents (142064 virtual) 2018-03-28 00:25:45,050 : INFO : 27 batches submitted to accumulate stats from 1728 documents (146798 virtual) 2018-03-28 00:25:45,184 : INFO : 28 batches submitted to accumulate stats from 1792 documents (151967 virtual) 2018-03-28 00:25:45,236 : INFO : 29 batches submitted to accumulate stats from 1856 documents (156931 virtual) 2018-03-28 00:25:45,292 : INFO : 30 batches submitted to accumulate stats from 1920 documents (162120 virtual) 2018-03-28 00:25:45,436 : INFO : 31 batches submitted to accumulate stats from 1984 documents (167657 virtual) 2018-03-28 00:25:45,492 : INFO : 32 batches submitted to accumulate stats from 2048 documents (173392 virtual) 2018-03-28 00:25:45,561 : INFO : 33 batches submitted to accumulate stats from 2112 documents (179271 virtual) 2018-03-28 00:25:45,676 : INFO : 34 batches submitted to accumulate stats from 2176 documents (184456 virtual) 2018-03-28 00:25:45,737 : INFO : 35 batches submitted to accumulate stats from 2240 documents (189712 virtual) 2018-03-28 00:25:45,811 : INFO : 36 batches submitted to accumulate stats from 2304 documents (194968 virtual) 2018-03-28 00:25:45,907 : INFO : 37 batches submitted to accumulate stats from 2368 documents (200416 virtual) 2018-03-28 00:25:46,013 : INFO : 38 batches submitted to accumulate stats from 2432 documents (206607 virtual) 2018-03-28 00:25:46,083 : INFO : 39 batches submitted to accumulate stats from 2496 documents (212256 virtual) 2018-03-28 00:25:46,171 : INFO : 40 batches submitted to accumulate stats from 2560 documents (217726 virtual) 2018-03-28 00:25:46,233 : INFO : 41 batches submitted to accumulate stats from 2624 documents (223132 virtual) 2018-03-28 00:25:46,333 : INFO : 42 batches submitted to accumulate stats from 2688 documents (228996 virtual) 2018-03-28 00:25:46,452 : INFO : 43 batches submitted to accumulate stats from 2752 documents (234484 virtual) 2018-03-28 00:25:46,494 : INFO : 44 batches submitted to accumulate stats from 2816 documents (240694 virtual) 2018-03-28 00:25:46,605 : INFO : 45 batches submitted to accumulate stats from 2880 documents (245727 virtual) 2018-03-28 00:25:46,713 : INFO : 46 batches submitted to accumulate stats from 2944 documents (250925 virtual) 2018-03-28 00:25:46,730 : INFO : 47 batches submitted to accumulate stats from 3008 documents (256354 virtual) 2018-03-28 00:25:46,843 : INFO : 48 batches submitted to accumulate stats from 3072 documents (261509 virtual) 2018-03-28 00:25:46,967 : INFO : 49 batches submitted to accumulate stats from 3136 documents (267781 virtual) 2018-03-28 00:25:47,009 : INFO : 50 batches submitted to accumulate stats from 3200 documents (272961 virtual) 2018-03-28 00:25:47,081 : INFO : 51 batches submitted to accumulate stats from 3264 documents (278022 virtual) 2018-03-28 00:25:47,229 : INFO : 52 batches submitted to accumulate stats from 3328 documents (283416 virtual) 2018-03-28 00:25:47,272 : INFO : 53 batches submitted to accumulate stats from 3392 documents (289548 virtual) 2018-03-28 00:25:47,351 : INFO : 54 batches submitted to accumulate stats from 3456 documents (294850 virtual) 2018-03-28 00:25:47,523 : INFO : 55 batches submitted to accumulate stats from 3520 documents (300906 virtual) 2018-03-28 00:25:47,571 : INFO : 56 batches submitted to accumulate stats from 3584 documents (306547 virtual) 2018-03-28 00:25:47,607 : INFO : 57 batches submitted to accumulate stats from 3648 documents (308658 virtual) 2018-03-28 00:25:47,936 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:47,942 : INFO : accumulator serialized 2018-03-28 00:25:48,080 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:48,107 : INFO : serializing accumulator to return to master... 2018-03-28 00:25:48,086 : INFO : accumulator serialized 2018-03-28 00:25:48,112 : INFO : accumulator serialized 2018-03-28 00:25:48,260 : INFO : 3 accumulators retrieved from output queue 2018-03-28 00:25:48,527 : INFO : accumulated word occurrence stats for 308867 virtual documents
Best number of topics¶
In [27]:
tmrest_meta_score.set_index('num_topic')
tmrest_meta_score.to_csv( os.path.join('../src', 'tmrest_meta', 'scores.combined.csv'))
In [40]:
f = plt.figure(figsize=(8,8))
ax1 = f.add_subplot(211)
ax1 = tmrest_meta_score.plot(x='num_topic',y='c_v',ax=ax1, color='b', linestyle='-')
ax1.set_ylabel('c_v coherence score')
ax1 = tmrest_meta_score.plot(x='num_topic', y='u_mass', ax=ax1, secondary_y=True, color='r', linestyle='-.')
ax1.set_ylabel('u_mass coherence score')
ax2 = f.add_subplot(212)
ax2 = tmrest_meta_score.plot(x='num_topic', y='c_npmi', ax=ax2, color='g', linestyle='--')
ax2.set_ylabel('c_npmi coherence score')
ax2 = tmrest_meta_score.plot(x='num_topic', y='c_uci',ax=ax2, secondary_y=True, color='m',linestyle=':')
ax2.set_ylabel('u_uci coherence score')
# df = tmrest_meta_score
# ax = tmrest_meta_score.plot(x='num_topic')
# best = df.c_v.argmax()
# ax.scatter(x=df.num_topic.iloc[best], y=df.iloc[best].c_v, c='g', marker='*', s=150)
Out[40]:
Text(0,0.5,'u_uci coherence score')
In [41]:
f.savefig('./IMG/tmrest_meta.png', fomat='png', bbox_inches='tight')
Top terms for lda model:¶
In [7]:
best_num_topics = 30
tmrest_meta_best_model = models.LdaModel.load(
'../src/tmrest_meta/ldamodel_nb_topics_{}'.format(best_num_topics))
2018-03-27 11:06:44,012 : INFO : loading LdaModel object from ../src/tmrest_meta/ldamodel_nb_topics_30 2018-03-27 11:06:44,418 : INFO : loading expElogbeta from ../src/tmrest_meta/ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-27 11:06:44,428 : INFO : setting ignored attribute id2word to None 2018-03-27 11:06:44,429 : INFO : setting ignored attribute dispatcher to None 2018-03-27 11:06:44,429 : INFO : setting ignored attribute state to None 2018-03-27 11:06:44,430 : INFO : loaded ../src/tmrest_meta/ldamodel_nb_topics_30 2018-03-27 11:06:44,430 : INFO : loading LdaModel object from ../src/tmrest_meta/ldamodel_nb_topics_30.state 2018-03-27 11:06:44,457 : INFO : loaded ../src/tmrest_meta/ldamodel_nb_topics_30.state
In [95]:
for i in range(best_num_topics):
print('TOPIC {}'.format(i))
eg = tmrest_meta_best_model.get_topic_terms(i, topn=15)
for (a,b) in eg:
print('{}\t\t{:.5f}'.format(dict_tmrest_meta[a], b))
print('\n\n')
TOPIC 0 system 0.02466 programming 0.02060 level 0.01960 high 0.01720 opencl 0.01625 gpus 0.01523 parallel 0.01413 application 0.01267 data 0.01247 type 0.01168 implementation 0.01079 unit 0.01040 high_level 0.00958 approach 0.00950 accelerator 0.00911 TOPIC 1 problem 0.01933 user 0.01819 search 0.01638 retrieval 0.01370 distribution 0.01362 algorithm 0.01164 result 0.01132 transducer 0.01084 exploratory 0.00816 given 0.00812 set 0.00782 relevance 0.00735 query 0.00638 show 0.00606 information 0.00587 TOPIC 2 query 0.04942 data 0.02290 database 0.02137 tree 0.01743 game 0.01566 class 0.01244 answer 0.00967 problem 0.00873 graph 0.00859 complexity 0.00798 show 0.00753 answering 0.00729 value 0.00688 one 0.00670 regular 0.00656 TOPIC 3 graph 0.05727 pattern 0.02301 data 0.02066 algorithm 0.01534 matching 0.01381 view 0.01000 problem 0.00967 analysis 0.00926 scene 0.00815 using 0.00814 based 0.00809 show 0.00774 real 0.00773 graph_pattern 0.00748 edge 0.00616 TOPIC 4 system 0.01686 performance 0.01526 code 0.01365 application 0.01202 program 0.00913 compiler 0.00901 software 0.00844 based 0.00759 acm 0.00680 approach 0.00674 technique 0.00662 instruction 0.00656 architecture 0.00613 present 0.00612 dynamic 0.00592 TOPIC 5 model 0.03771 parameter 0.01228 kernel 0.01197 control 0.00914 proposed 0.00905 using 0.00901 approach 0.00892 gait 0.00858 dynamic 0.00841 based 0.00789 method 0.00760 data 0.00748 input 0.00711 walking 0.00624 linear 0.00546 TOPIC 6 security 0.01923 workflow 0.01582 protocol 0.01546 system 0.01414 privacy 0.01398 application 0.01375 cloud 0.01361 computing 0.01178 data 0.00979 process 0.00817 service 0.00775 attack 0.00693 computation 0.00595 based 0.00582 policy 0.00580 TOPIC 7 network 0.02706 model 0.02564 feature 0.02530 neural 0.02342 acoustic 0.01927 speech 0.01503 using 0.01446 deep 0.01328 ieee 0.01226 training 0.01169 based 0.01148 recognition 0.01072 neural_network 0.00906 data 0.00833 signal 0.00778 TOPIC 8 image 0.04792 object 0.03588 vision 0.01608 class 0.01481 video 0.01143 computer 0.01072 method 0.01066 computer_vision 0.01031 classification 0.00882 segmentation 0.00812 model 0.00787 visual 0.00739 annotation 0.00711 training 0.00660 scene 0.00636 TOPIC 9 memory 0.02327 program 0.01672 performance 0.01552 cache 0.01434 sketch 0.01307 level 0.01257 core 0.01158 processor 0.01095 parallel 0.00911 grained 0.00894 fine 0.00881 high 0.00782 fine_grained 0.00745 storage 0.00735 design 0.00710 TOPIC 10 translation 0.06703 machine 0.03783 system 0.02382 machine_translation 0.02378 task 0.01904 language 0.01025 paper 0.01016 statistical 0.00990 proceeding 0.00938 english 0.00893 evaluation 0.00885 mt 0.00861 shared 0.00860 data 0.00827 workshop 0.00815 TOPIC 11 model 0.02188 data 0.01267 source 0.01256 time 0.00959 decision 0.00724 task 0.00710 information 0.00653 code 0.00628 result 0.00618 subject 0.00602 speed 0.00589 change 0.00540 using 0.00526 reliability 0.00514 annotation 0.00471 TOPIC 12 semantic 0.01969 representation 0.01788 model 0.01701 data 0.01585 learning 0.01565 attribute 0.01505 category 0.01404 domain 0.01331 visual 0.01301 map 0.01073 space 0.01014 recognition 0.01013 feature 0.01002 mapping 0.00825 alignment 0.00715 TOPIC 13 cell 0.01417 neuron 0.01187 protein 0.00996 neural 0.00865 synaptic 0.00854 activity 0.00818 response 0.00806 brain 0.00792 gene 0.00792 information 0.00709 population 0.00661 analysis 0.00650 model 0.00631 expression 0.00627 data 0.00587 TOPIC 14 system 0.06692 model 0.03809 modelling 0.02039 stochastic 0.01723 process 0.01454 analysis 0.01019 collective 0.01003 approach 0.00964 adaptive 0.00897 behaviour 0.00817 simulation 0.00805 dynamic 0.00759 method 0.00744 formal 0.00741 quantitative 0.00740 TOPIC 15 network 0.04412 data 0.03852 performance 0.01089 mobile 0.00841 application 0.00734 wireless 0.00703 latency 0.00617 ieee 0.00615 using 0.00611 based 0.00595 measurement 0.00569 access 0.00540 architecture 0.00524 service 0.00482 high 0.00480 TOPIC 16 algorithm 0.04014 problem 0.02192 time 0.01566 optimal 0.01497 polynomial 0.01333 learning 0.01319 variable 0.01227 function 0.01049 complexity 0.01002 probability 0.01001 value 0.00981 degree 0.00818 bound 0.00759 vector 0.00731 approximate 0.00707 TOPIC 17 social 0.02117 user 0.01639 web 0.01393 tweet 0.01081 content 0.01014 using 0.00908 different 0.00903 information 0.00890 international 0.00873 data 0.00725 story 0.00716 conference 0.00700 medium 0.00686 arabic 0.00654 topic 0.00637 TOPIC 18 user 0.02813 motion 0.01847 system 0.01816 search 0.01198 character 0.01033 interface 0.00919 interactive 0.00905 environment 0.00899 interaction 0.00845 information 0.00843 virtual 0.00830 computer 0.00767 fish 0.00744 based 0.00729 behavior 0.00688 TOPIC 19 language 0.03041 computational 0.02043 association 0.02038 model 0.01964 linguistics 0.01842 word 0.01696 association_computational 0.01417 proceeding 0.01190 sentence 0.00935 natural 0.00934 text 0.00906 semantic 0.00808 natural_language 0.00785 association_computational_linguistics 0.00781 based 0.00742 TOPIC 20 quantum 0.04119 theory 0.01483 computation 0.01461 algebra 0.01160 protocol 0.01112 category 0.00999 state 0.00993 classical 0.00988 one 0.00886 signature 0.00869 scheme 0.00820 construction 0.00690 measurement 0.00637 space 0.00635 structure 0.00617 TOPIC 21 speech 0.07716 synthesis 0.02239 voice 0.01933 speaker 0.01711 system 0.01357 speech_synthesis 0.01337 synthetic 0.01199 communication 0.00885 paper 0.00833 using 0.00831 data 0.00825 based 0.00810 speech_communication 0.00796 synthetic_speech 0.00724 text 0.00701 TOPIC 22 robot 0.02657 control 0.02021 task 0.01623 provenance 0.01299 planning 0.01240 ieee 0.01156 environment 0.00988 motion 0.00971 humanoid 0.00967 system 0.00926 dynamic 0.00901 based 0.00844 learning 0.00842 policy 0.00836 method 0.00814 TOPIC 23 language 0.01836 reasoning 0.01653 ontology 0.01651 knowledge 0.01532 rule 0.01480 proceeding 0.01257 domain 0.01216 intelligence 0.01211 artificial 0.01114 logic 0.00966 representation 0.00956 workshop 0.00792 international 0.00773 artificial_intelligence 0.00770 semantics 0.00740 TOPIC 24 based 0.01299 model 0.01297 net 0.01025 network 0.00992 method 0.00849 error 0.00797 show 0.00790 fusion 0.00724 matter 0.00705 face 0.00685 cost 0.00647 chemical 0.00630 state 0.00615 petri_net 0.00595 petri 0.00595 TOPIC 25 dialogue 0.01785 child 0.01709 people 0.01351 human 0.01280 learning 0.01152 interaction 0.01132 study 0.01076 cognitive 0.00963 behaviour 0.00826 task 0.00826 visual 0.00825 participant 0.00812 malware 0.00668 result 0.00665 design 0.00664 TOPIC 26 model 0.03422 method 0.02105 inference 0.01684 learning 0.01588 bayesian 0.01273 approach 0.01132 distribution 0.01007 data 0.00926 machine 0.00882 probabilistic 0.00840 machine_learning 0.00822 prior 0.00747 based 0.00720 sampling 0.00706 set 0.00689 TOPIC 27 acm 0.04432 proceeding 0.02264 new 0.02255 york 0.01736 new_york 0.01702 usa 0.01581 system 0.01566 ny 0.01531 ny_usa 0.01522 agent 0.01463 conference 0.01176 programming 0.01130 language 0.01076 international 0.00963 design 0.00938 TOPIC 28 springer 0.05071 international 0.03480 berlin 0.02434 heidelberg 0.02338 publishing 0.02200 conference 0.01527 proceeding 0.01482 berlin_heidelberg 0.01284 proof 0.01224 system 0.01091 springer_berlin 0.01069 springer_international 0.01061 language 0.00977 international_publishing 0.00972 paper 0.00929 TOPIC 29 energy 0.02417 research 0.02130 project 0.01277 social 0.00978 technology 0.00938 design 0.00878 identification 0.00861 practice 0.00798 paper 0.00767 data 0.00749 community 0.00734 science 0.00726 challenge 0.00725 approach 0.00695 future 0.00660
In [8]:
tmrest_meta_best_model.save('../src/tmrest_meta/res/best_ldamodel')
2018-03-27 11:07:19,002 : INFO : saving LdaState object under ../src/tmrest_meta/res/best_ldamodel.state, separately None 2018-03-27 11:07:19,010 : INFO : saved ../src/tmrest_meta/res/best_ldamodel.state 2018-03-27 11:07:19,012 : INFO : saving LdaModel object under ../src/tmrest_meta/res/best_ldamodel, separately ['expElogbeta', 'sstats'] 2018-03-27 11:07:19,012 : INFO : storing np array 'expElogbeta' to ../src/tmrest_meta/res/best_ldamodel.expElogbeta.npy 2018-03-27 11:07:19,015 : INFO : not storing attribute dispatcher 2018-03-27 11:07:19,016 : INFO : not storing attribute state 2018-03-27 11:07:19,016 : INFO : not storing attribute id2word 2018-03-27 11:07:19,018 : INFO : saved ../src/tmrest_meta/res/best_ldamodel
Get $C_V$ Scores¶
In [123]:
cv_score = tmrest_meta_best_model.top_topics(
corpus_tmrest_meta,
text_tmrest_meta,
dict_tmrest_meta,
coherence='c_v',
topn=15)
2018-03-29 10:27:08,536 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-29 10:27:12,954 : INFO : serializing accumulator to return to master... 2018-03-29 10:27:12,980 : INFO : serializing accumulator to return to master... 2018-03-29 10:27:13,004 : INFO : serializing accumulator to return to master... 2018-03-29 10:27:12,960 : INFO : accumulator serialized 2018-03-29 10:27:12,986 : INFO : accumulator serialized 2018-03-29 10:27:13,010 : INFO : accumulator serialized 2018-03-29 10:27:13,190 : INFO : 3 accumulators retrieved from output queue 2018-03-29 10:27:13,379 : INFO : accumulated word occurrence stats for 48403 virtual documents
In [124]:
topics= [19, 21, 27, 7, 28, 8,
0, 23, 13, 10, 16,
17, 12, 4, 15, 26,
14, 22, 9, 29, 20,
6, 2, 18, 25, 5,
1, 3, 24, 11, ]
labels = ['Computational Linguistics', 'Speech Synthesis', 'IRR', 'Neural Network', 'IRR', 'Computer Vision',
'Parallel Programming', 'Reasoning (?)','Bioinformatics','Machine Translation','Algorithms',
'Social Media', '??', 'Compiler', 'Wireless Communication', 'Machine Learning',
'Computer Simulation','Robot Control', 'Computer System', '??', 'Quantum Computing',
'Computer Security', 'Database', 'Human Computer Interaction', 'Interaction System (?)', 'Modelling (?)',
'Information Retrieval', 'Graph Theory', '??', '??']
assert len(topics) == len(labels)
for i, (a,b) in enumerate(cv_score):
nb_topic = topics[i]+1
topic_label = labels[i]
_str = "\multirow{3}{*}{" + str(nb_topic) + "} & \multirow{3}{*}{" + "{:.3f}".format(b) + "} "
start=True
for j, (prob, word) in enumerate(a):
_j = j+1
_str += " & " + str(word.replace('_', '\_'))
if _j%5 == 0:
if start:
start=False
_str += " & \multirow{3}{*}{" + str(topic_label) + "} \\\\ \n &"
elif _j == 15:
_str += " \\\\ \\midrule \n"
else:
_str += " \\\\ \n &"
print(_str)
\multirow{3}{*}{20} & \multirow{3}{*}{0.741} & language & computational & association & model & linguistics & \multirow{3}{*}{Computational Linguistics} \\
& & word & association\_computational & proceeding & sentence & natural \\
& & text & semantic & natural\_language & association\_computational\_linguistics & based \\ \midrule
\multirow{3}{*}{22} & \multirow{3}{*}{0.687} & speech & synthesis & voice & speaker & system & \multirow{3}{*}{Speech Synthesis} \\
& & speech\_synthesis & synthetic & communication & paper & using \\
& & data & based & speech\_communication & synthetic\_speech & text \\ \midrule
\multirow{3}{*}{28} & \multirow{3}{*}{0.686} & acm & proceeding & new & york & new\_york & \multirow{3}{*}{IRR} \\
& & usa & system & ny & ny\_usa & agent \\
& & conference & programming & language & international & design \\ \midrule
\multirow{3}{*}{8} & \multirow{3}{*}{0.662} & network & model & feature & neural & acoustic & \multirow{3}{*}{Neural Network} \\
& & speech & using & deep & ieee & training \\
& & based & recognition & neural\_network & data & signal \\ \midrule
\multirow{3}{*}{29} & \multirow{3}{*}{0.644} & springer & international & berlin & heidelberg & publishing & \multirow{3}{*}{IRR} \\
& & conference & proceeding & berlin\_heidelberg & proof & system \\
& & springer\_berlin & springer\_international & language & international\_publishing & paper \\ \midrule
\multirow{3}{*}{9} & \multirow{3}{*}{0.642} & image & object & vision & class & video & \multirow{3}{*}{Computer Vision} \\
& & computer & method & computer\_vision & classification & segmentation \\
& & model & visual & annotation & training & scene \\ \midrule
\multirow{3}{*}{1} & \multirow{3}{*}{0.636} & system & programming & level & high & opencl & \multirow{3}{*}{Parallel Programming} \\
& & gpus & parallel & application & data & type \\
& & implementation & unit & high\_level & approach & accelerator \\ \midrule
\multirow{3}{*}{24} & \multirow{3}{*}{0.622} & language & reasoning & ontology & knowledge & rule & \multirow{3}{*}{Reasoning (?)} \\
& & proceeding & domain & intelligence & artificial & logic \\
& & representation & workshop & international & artificial\_intelligence & semantics \\ \midrule
\multirow{3}{*}{14} & \multirow{3}{*}{0.594} & cell & neuron & protein & neural & synaptic & \multirow{3}{*}{Bioinformatics} \\
& & activity & response & brain & gene & information \\
& & population & analysis & model & expression & data \\ \midrule
\multirow{3}{*}{11} & \multirow{3}{*}{0.577} & translation & machine & system & machine\_translation & task & \multirow{3}{*}{Machine Translation} \\
& & language & paper & statistical & proceeding & english \\
& & evaluation & mt & shared & data & workshop \\ \midrule
\multirow{3}{*}{17} & \multirow{3}{*}{0.573} & algorithm & problem & time & optimal & polynomial & \multirow{3}{*}{Algorithms} \\
& & learning & variable & function & complexity & probability \\
& & value & degree & bound & vector & approximate \\ \midrule
\multirow{3}{*}{18} & \multirow{3}{*}{0.553} & social & user & web & tweet & content & \multirow{3}{*}{Social Media} \\
& & using & different & information & international & data \\
& & story & conference & medium & arabic & topic \\ \midrule
\multirow{3}{*}{13} & \multirow{3}{*}{0.546} & semantic & representation & model & data & learning & \multirow{3}{*}{??} \\
& & attribute & category & domain & visual & map \\
& & space & recognition & feature & mapping & alignment \\ \midrule
\multirow{3}{*}{5} & \multirow{3}{*}{0.539} & system & performance & code & application & program & \multirow{3}{*}{Compiler} \\
& & compiler & software & based & acm & approach \\
& & technique & instruction & architecture & present & dynamic \\ \midrule
\multirow{3}{*}{16} & \multirow{3}{*}{0.526} & network & data & performance & mobile & application & \multirow{3}{*}{Wireless Communication} \\
& & wireless & latency & ieee & using & based \\
& & measurement & access & architecture & service & high \\ \midrule
\multirow{3}{*}{27} & \multirow{3}{*}{0.500} & model & method & inference & learning & bayesian & \multirow{3}{*}{Machine Learning} \\
& & approach & distribution & data & machine & probabilistic \\
& & machine\_learning & prior & based & sampling & set \\ \midrule
\multirow{3}{*}{15} & \multirow{3}{*}{0.489} & system & model & modelling & stochastic & process & \multirow{3}{*}{Computer Simulation} \\
& & analysis & collective & approach & adaptive & behaviour \\
& & simulation & dynamic & method & formal & quantitative \\ \midrule
\multirow{3}{*}{23} & \multirow{3}{*}{0.473} & robot & control & task & provenance & planning & \multirow{3}{*}{Robot Control} \\
& & ieee & environment & motion & humanoid & system \\
& & dynamic & based & learning & policy & method \\ \midrule
\multirow{3}{*}{10} & \multirow{3}{*}{0.467} & memory & program & performance & cache & sketch & \multirow{3}{*}{Computer System} \\
& & level & core & processor & parallel & grained \\
& & fine & high & fine\_grained & storage & design \\ \midrule
\multirow{3}{*}{30} & \multirow{3}{*}{0.463} & energy & research & project & social & technology & \multirow{3}{*}{??} \\
& & design & identification & practice & paper & data \\
& & community & science & challenge & approach & future \\ \midrule
\multirow{3}{*}{21} & \multirow{3}{*}{0.417} & quantum & theory & computation & algebra & protocol & \multirow{3}{*}{Quantum Computing} \\
& & category & state & classical & one & signature \\
& & scheme & construction & measurement & space & structure \\ \midrule
\multirow{3}{*}{7} & \multirow{3}{*}{0.415} & security & workflow & protocol & system & privacy & \multirow{3}{*}{Computer Security} \\
& & application & cloud & computing & data & process \\
& & service & attack & computation & based & policy \\ \midrule
\multirow{3}{*}{3} & \multirow{3}{*}{0.409} & query & data & database & tree & game & \multirow{3}{*}{Database} \\
& & class & answer & problem & graph & complexity \\
& & show & answering & value & one & regular \\ \midrule
\multirow{3}{*}{19} & \multirow{3}{*}{0.393} & user & motion & system & search & character & \multirow{3}{*}{Human Computer Interaction} \\
& & interface & interactive & environment & interaction & information \\
& & virtual & computer & fish & based & behavior \\ \midrule
\multirow{3}{*}{26} & \multirow{3}{*}{0.373} & dialogue & child & people & human & learning & \multirow{3}{*}{Interaction System (?)} \\
& & interaction & study & cognitive & behaviour & task \\
& & visual & participant & malware & result & design \\ \midrule
\multirow{3}{*}{6} & \multirow{3}{*}{0.369} & model & parameter & kernel & control & proposed & \multirow{3}{*}{Modelling (?)} \\
& & using & approach & gait & dynamic & based \\
& & method & data & input & walking & linear \\ \midrule
\multirow{3}{*}{2} & \multirow{3}{*}{0.363} & problem & user & search & retrieval & distribution & \multirow{3}{*}{Information Retrieval} \\
& & algorithm & result & transducer & exploratory & given \\
& & set & relevance & query & show & information \\ \midrule
\multirow{3}{*}{4} & \multirow{3}{*}{0.340} & graph & pattern & data & algorithm & matching & \multirow{3}{*}{Graph Theory} \\
& & view & problem & analysis & scene & using \\
& & based & show & real & graph\_pattern & edge \\ \midrule
\multirow{3}{*}{25} & \multirow{3}{*}{0.285} & based & model & net & network & method & \multirow{3}{*}{??} \\
& & error & show & fusion & matter & face \\
& & cost & chemical & state & petri\_net & petri \\ \midrule
\multirow{3}{*}{12} & \multirow{3}{*}{0.262} & model & data & source & time & decision & \multirow{3}{*}{??} \\
& & task & information & code & result & subject \\
& & speed & change & using & reliability & annotation \\ \midrule
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:25: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:25: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:25: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:25: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:25: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:25: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:25: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:25: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:25: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:25: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:25: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:17: DeprecationWarning: invalid escape sequence \m
<input>:21: DeprecationWarning: invalid escape sequence \_
<input>:25: DeprecationWarning: invalid escape sequence \m
<ipython-input-124-cdf4468d27be>:17: DeprecationWarning: invalid escape sequence \m
_str = "\multirow{3}{*}{" + str(nb_topic) + "} & \multirow{3}{*}{" + "{:.3f}".format(b) + "} "
<ipython-input-124-cdf4468d27be>:17: DeprecationWarning: invalid escape sequence \m
_str = "\multirow{3}{*}{" + str(nb_topic) + "} & \multirow{3}{*}{" + "{:.3f}".format(b) + "} "
<ipython-input-124-cdf4468d27be>:21: DeprecationWarning: invalid escape sequence \_
_str += " & " + str(word.replace('_', '\_'))
<ipython-input-124-cdf4468d27be>:25: DeprecationWarning: invalid escape sequence \m
_str += " & \multirow{3}{*}{" + str(topic_label) + "} \\\\ \n &"
In [63]:
probs = []
for (a,b) in cv_score:
probs.append(b)
mean_cv_score = np.mean(probs)
median_cv_score = np.median(probs)
In [64]:
sns.distplot(probs)
/Users/weiting/miniconda3/envs/infnet3/lib/python3.6/site-packages/statsmodels/nonparametric/kde.py:475: DeprecationWarning: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer. grid,delta = np.linspace(a,b,gridsize,retstep=True)
Out[64]:
<matplotlib.axes._subplots.AxesSubplot at 0x1a6515d780>
In [65]:
print(mean_cv_score, median_cv_score)
0.5082615420437593 0.5131397362440104
Visualisation¶
In [55]:
tmrest_meta_best_model = models.LdaModel.load('../src/tmrest_meta/res/best_ldamodel')
2018-03-28 20:23:58,627 : INFO : loading LdaModel object from ../src/tmrest_meta/res/best_ldamodel 2018-03-28 20:23:58,679 : INFO : loading expElogbeta from ../src/tmrest_meta/res/best_ldamodel.expElogbeta.npy with mmap=None 2018-03-28 20:23:58,698 : INFO : setting ignored attribute dispatcher to None 2018-03-28 20:23:58,698 : INFO : setting ignored attribute state to None 2018-03-28 20:23:58,699 : INFO : setting ignored attribute id2word to None 2018-03-28 20:23:58,699 : INFO : loaded ../src/tmrest_meta/res/best_ldamodel 2018-03-28 20:23:58,700 : INFO : loading LdaModel object from ../src/tmrest_meta/res/best_ldamodel.state 2018-03-28 20:23:58,727 : INFO : loaded ../src/tmrest_meta/res/best_ldamodel.state
In [57]:
vis_data = gensimvis.prepare(
tmrest_meta_best_model,
corpus_tmrest_meta,
dict_tmrest_meta,
sort_topics=False,
mds='mmds')
/Users/weiting/miniconda3/envs/infnet3/lib/python3.6/site-packages/pyLDAvis/_prepare.py:387: DeprecationWarning: .ix is deprecated. Please use .loc for label based indexing or .iloc for positional indexing See the documentation here: http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated topic_term_dists = topic_term_dists.ix[topic_order]
In [58]:
pyLDAvis.display(vis_data)
Out[58]:
In [67]:
# save in html for viewing
pyLDAvis.save_html(vis_data, open('../src/tmrest_meta/res/ldavis.mds.html', 'w'))
tm-dblp : topic modelling using dblp¶
In [8]:
DATA_DIR = '../../data/data_dblp'
In [9]:
dict_dblp = Dictionary.load(os.path.join(DATA_DIR, 'corpora', 'dictionary.dblp.1997-2017'))
2018-03-27 11:15:08,823 : INFO : loading Dictionary object from ../../data/data_dblp/corpora/dictionary.dblp.1997-2017 2018-03-27 11:15:08,898 : INFO : loaded ../../data/data_dblp/corpora/dictionary.dblp.1997-2017
In [ ]:
dblp_toks = pd.read_pickle(os.path.join(DATA_DIR,'toks','toks.dblp.1997-2017.pkl'))
dblp_toks.info()
In [9]:
# convert dblp_toks to BOW:
dblp_toks['bow'] = dblp_toks['toks'].apply(dict_dblp.doc2bow)
corpus_dblp = dblp_toks.bow.tolist()a
text_dblp = dblp_toks.toks.tolist()
2018-02-06 12:38:44,331 : INFO : loading Dictionary object from ../../data/data_dblp/corpora/dictionary.dblp.1997-2017 2018-02-06 12:38:44,399 : INFO : loaded ../../data/data_dblp/corpora/dictionary.dblp.1997-2017
Top terms for LDA¶
In [5]:
tmdblp = models.LdaModel.load('../src/tmdblp/tm_dblp')
2018-03-27 11:14:04,472 : INFO : loading LdaModel object from ../src/tmdblp/tm_dblp 2018-03-27 11:14:04,536 : INFO : loading expElogbeta from ../src/tmdblp/tm_dblp.expElogbeta.npy with mmap=None 2018-03-27 11:14:05,113 : INFO : setting ignored attribute id2word to None 2018-03-27 11:14:05,114 : INFO : setting ignored attribute dispatcher to None 2018-03-27 11:14:05,115 : INFO : setting ignored attribute state to None 2018-03-27 11:14:05,115 : INFO : loaded ../src/tmdblp/tm_dblp 2018-03-27 11:14:05,116 : INFO : loading LdaModel object from ../src/tmdblp/tm_dblp.state 2018-03-27 11:14:06,282 : INFO : loaded ../src/tmdblp/tm_dblp.state
In [10]:
for i in range(100):
print('TOPIC {}'.format(i))
eg = tmdblp.get_topic_terms(i, topn=15)
for (a,b) in eg:
print('{}\t\t{:.5f}'.format(dict_dblp[a], b))
print('\n\n')
TOPIC 0 image 0.02606 surface 0.02481 method 0.01569 shape 0.01562 resolution 0.01405 computer 0.01154 reconstruction 0.01066 graphic 0.01066 point 0.00959 technique 0.00947 depth 0.00895 field 0.00842 scene 0.00777 approach 0.00715 algorithm 0.00692 TOPIC 1 rate 0.04649 spectrum 0.03644 interference 0.03552 radio 0.02956 coding 0.01569 user 0.01510 cognitive 0.01460 bit 0.01089 transmission 0.00904 cognitive_radio 0.00876 channel 0.00858 indoor 0.00858 outage 0.00846 band 0.00828 frequency 0.00792 TOPIC 2 agent 0.07083 knowledge 0.04567 intelligence 0.02859 artificial 0.02553 multi 0.01745 artificial_intelligence 0.01661 conference 0.01175 multi_agent 0.01102 domain 0.00879 approach 0.00823 environment 0.00785 framework 0.00781 intelligent 0.00704 self 0.00651 reasoning 0.00650 TOPIC 3 interface 0.05338 display 0.03126 user 0.02690 hand 0.02357 device 0.02304 gesture 0.01501 input 0.01243 screen 0.01024 finger 0.00833 touch 0.00798 tactile 0.00674 pc 0.00641 computer 0.00624 interaction 0.00601 grasp 0.00583 TOPIC 4 signal 0.06567 speech 0.05176 processing 0.02361 acoustic 0.01937 signal_processing 0.01927 international 0.01448 conference 0.01400 recognition 0.01312 frequency 0.01174 speaker 0.01011 estimation 0.01007 international_conference 0.00961 acoustic_speech 0.00927 noise 0.00857 method 0.00833 TOPIC 5 model 0.10764 simulation 0.02820 modeling 0.02569 process 0.02369 design 0.01871 approach 0.01722 requirement 0.01239 analysis 0.00915 framework 0.00860 driven 0.00804 application 0.00801 modelling 0.00737 workflow 0.00696 level 0.00695 tool 0.00693 TOPIC 6 virtual 0.05204 game 0.05173 interaction 0.02572 interactive 0.02228 environment 0.01796 player 0.01483 quantum 0.01336 reality 0.01313 computer 0.01265 user 0.01134 design 0.01046 physical 0.00971 human 0.00953 movement 0.00811 world 0.00789 TOPIC 7 safety 0.04729 law 0.02439 self 0.01946 critical 0.01442 legal 0.01281 predictor 0.01134 fall 0.01122 compliance 0.01093 branch 0.01021 damage 0.00906 driving 0.00855 vertical 0.00815 ion 0.00693 incentive 0.00669 forensic 0.00655 TOPIC 8 student 0.03671 computer 0.03491 science 0.02793 computer_science 0.01978 education 0.01774 course 0.01596 university 0.01226 technology 0.01181 lecture 0.01066 teaching 0.00959 note 0.00896 study 0.00864 child 0.00858 educational 0.00841 lecture_note 0.00831 TOPIC 9 inc 0.02416 wiley 0.02367 periodical 0.01651 wiley_periodical 0.01388 metal 0.01347 oxide 0.00908 subgraphs 0.00892 wiley_periodical_inc 0.00824 abstract 0.00713 tournament 0.00626 extremal 0.00614 japan 0.00599 sir 0.00590 glucose 0.00574 hypergraphs 0.00547 TOPIC 10 grid 0.14542 metric 0.10638 soft 0.04183 smart 0.02100 fingerprint 0.01877 passing 0.01818 computing 0.01777 message_passing 0.01174 electric 0.01131 application 0.00968 message 0.00944 smart_grid 0.00870 shadow 0.00838 soft_computing 0.00812 copy 0.00799 TOPIC 11 graph 0.08747 bound 0.02540 edge 0.02232 set 0.01868 vertex 0.01848 number 0.01677 lower 0.00960 upper 0.00903 discrete 0.00818 algorithm 0.00807 connected 0.00800 tree 0.00771 show 0.00734 degree 0.00731 two 0.00715 TOPIC 12 state 0.02615 stability 0.02469 condition 0.01825 chain 0.01814 transition 0.01537 stable 0.01369 dynamic 0.01319 sub 0.01313 periodic 0.01149 model 0.01041 markov 0.00988 dynamical 0.00911 time 0.00888 sufficient 0.00874 chaotic 0.00797 TOPIC 13 wave 0.02129 alignment 0.01773 element 0.01686 matrix 0.01604 method 0.01554 inverse 0.01297 array 0.01254 beam 0.00982 interpolation 0.00748 ct 0.00723 eigenvalue 0.00703 covariance 0.00697 deformation 0.00660 elastic 0.00599 field 0.00569 TOPIC 14 face 0.12055 class 0.03956 negative 0.02162 face_recognition 0.02002 recognition 0.01746 factorization 0.01333 drawing 0.01125 non 0.00863 facial 0.00803 nc 0.00741 balanced 0.00594 non_negative 0.00589 lag 0.00584 slicing 0.00536 projective 0.00535 TOPIC 15 image 0.11259 feature 0.03649 method 0.02131 detection 0.02056 segmentation 0.01682 recognition 0.01317 visual 0.01251 proposed 0.01101 region 0.00903 extraction 0.00855 analysis 0.00846 imaging 0.00843 color 0.00798 information 0.00706 local 0.00680 TOPIC 16 wavelet 0.04116 chinese 0.02997 comparative 0.02350 pacific 0.02105 asia 0.01993 gradient 0.01674 join 0.01229 asia_pacific 0.01221 bp 0.01124 competition 0.01115 fir 0.01113 comparative_study 0.01062 china 0.01013 complex 0.00967 container 0.00955 TOPIC 17 social 0.03943 information 0.02752 online 0.01825 research 0.01775 community 0.01735 study 0.01678 medium 0.01138 journal 0.00883 factor 0.00863 use 0.00839 relationship 0.00780 electronic 0.00697 influence 0.00687 finding 0.00666 individual 0.00632 TOPIC 18 human 0.02401 user 0.01807 task 0.01665 interaction 0.01146 study 0.01071 activity 0.01004 behavior 0.00806 cognitive 0.00774 visual 0.00625 information 0.00611 participant 0.00593 work 0.00536 different 0.00518 design 0.00515 perception 0.00497 TOPIC 19 problem 0.07877 algorithm 0.06232 time 0.02695 optimal 0.01829 solution 0.01510 scheduling 0.01321 constraint 0.01318 heuristic 0.01131 case 0.00904 cost 0.00876 number 0.00793 show 0.00729 complexity 0.00712 two 0.00681 solve 0.00667 TOPIC 20 security 0.06603 attack 0.03206 network 0.02616 protocol 0.01634 ip 0.01304 privacy 0.01262 secure 0.01099 authentication 0.01054 communication 0.01036 protection 0.01033 mechanism 0.00993 information 0.00751 application 0.00684 threat 0.00675 mobility 0.00664 TOPIC 21 learning 0.20942 machine 0.02340 machine_learning 0.02293 visualization 0.01641 training 0.01582 learner 0.01541 online 0.01110 learn 0.01015 approach 0.00685 international 0.00663 learned 0.00583 research 0.00583 conference 0.00569 opinion 0.00545 study 0.00542 TOPIC 22 trace 0.02146 patch 0.01523 stock 0.01504 forward 0.01411 hole 0.01239 agreement 0.01208 exchange 0.01136 protocol 0.01064 backward 0.00997 ice 0.00924 cc 0.00906 secrecy 0.00779 violation 0.00759 party 0.00714 em 0.00712 TOPIC 23 model 0.04262 neural 0.02067 question 0.02024 network 0.01911 hidden 0.01907 neuron 0.01611 markov 0.01337 activity 0.01282 artificial 0.01157 answer 0.01139 artificial_neural 0.01049 artificial_neural_network 0.00802 input 0.00795 hidden_markov 0.00726 answering 0.00704 TOPIC 24 memory 0.10991 log 0.03845 read 0.01868 write 0.01358 flash 0.00959 macro 0.00881 access 0.00840 associative 0.00794 cell 0.00777 bit 0.00754 asymmetric 0.00730 locality 0.00720 store 0.00703 sram 0.00654 time 0.00641 TOPIC 25 color 0.03346 window 0.02856 segment 0.02405 volume 0.02322 contour 0.01964 ant 0.01407 cyber 0.01160 colony 0.01019 ray 0.00973 algorithm 0.00961 slice 0.00958 sa 0.00846 line 0.00790 pressure 0.00787 cardiac 0.00752 TOPIC 26 video 0.07251 tracking 0.04205 frame 0.02026 model 0.01077 visual 0.01027 track 0.01023 target 0.00842 positioning 0.00787 gps 0.00783 sequence 0.00728 surveillance 0.00711 animation 0.00680 method 0.00617 shape 0.00587 disaster 0.00572 TOPIC 27 architecture 0.03228 hardware 0.02814 core 0.01522 implementation 0.01421 application 0.01278 embedded 0.01275 design 0.01272 processor 0.01060 fpga 0.01029 performance 0.00960 high 0.00897 time 0.00875 reconfigurable 0.00793 module 0.00781 platform 0.00765 TOPIC 28 verification 0.07786 formal 0.03113 validation 0.02150 r 0.01446 method 0.01285 discrete_event 0.00910 assertion 0.00889 correctness 0.00857 reserved 0.00779 red 0.00730 model 0.00704 narrative 0.00679 all 0.00667 chemical 0.00648 dm 0.00643 TOPIC 29 filter 0.02692 algorithm 0.01954 proposed 0.01773 method 0.01637 coding 0.01319 compression 0.01313 image 0.01277 signal 0.01252 processing 0.01025 video 0.01007 block 0.00920 quality 0.00871 transform 0.00827 adaptive 0.00728 source 0.00711 TOPIC 30 sensing 0.03712 remote 0.03190 remote_sensing 0.01780 radar 0.01551 al 0.01453 geoscience 0.01346 et 0.01320 et_al 0.01161 water 0.01142 band 0.01128 surface 0.01085 spectral 0.00845 area 0.00828 symposium 0.00793 resolution 0.00761 TOPIC 31 time 0.15168 real 0.10640 real_time 0.07249 monitoring 0.01848 traffic 0.01658 temporal 0.01414 road 0.01115 application 0.00924 dynamic 0.00852 real_world 0.00812 driver 0.00680 spatio 0.00593 spatio_temporal 0.00567 world 0.00557 control 0.00494 TOPIC 32 allocation 0.05189 resource 0.04411 station 0.02746 base 0.02437 cell 0.01994 cellular 0.01583 power 0.01367 resource_allocation 0.01319 base_station 0.01261 network 0.01259 cross 0.01159 wlan 0.00929 user 0.00899 cross_layer 0.00631 proposed 0.00558 TOPIC 33 file 0.03981 multimedia 0.03978 storage 0.03305 streaming 0.01900 video 0.01821 application 0.01509 disk 0.01237 server 0.01188 coverage 0.01151 medium 0.00969 performance 0.00904 trade 0.00866 high 0.00762 quality 0.00666 network 0.00586 TOPIC 34 noise 0.07544 measurement 0.04853 signal 0.01533 diffusion 0.01091 white 0.01044 ieee 0.00912 transaction 0.00879 coupling 0.00824 ieee_transaction 0.00773 denoising 0.00652 instrumentation 0.00647 electrode 0.00641 tuning 0.00565 performance 0.00537 radiation 0.00534 TOPIC 35 sensor 0.13513 flow 0.07374 target 0.02121 sensing 0.01483 data 0.01170 forest 0.01137 measurement 0.01119 array 0.01104 monitoring 0.01020 detection 0.00835 terrain 0.00824 sink 0.00814 fire 0.00577 used 0.00577 c 0.00541 TOPIC 36 web 0.07395 service 0.07202 semantic 0.02903 ontology 0.02729 application 0.01304 composition 0.01155 information 0.01147 approach 0.00987 domain 0.00935 concept 0.00798 knowledge 0.00767 resource 0.00754 semantic_web 0.00742 framework 0.00729 integration 0.00666 TOPIC 37 data 0.18802 mining 0.02089 information 0.01309 analysis 0.01260 detection 0.01173 large 0.00790 technique 0.00749 discovery 0.00722 approach 0.00667 event 0.00648 knowledge 0.00627 source 0.00595 database 0.00576 set 0.00560 application 0.00523 TOPIC 38 vector 0.05865 classification 0.04063 machine 0.03901 support 0.02211 method 0.01991 regression 0.01876 svm 0.01791 support_vector 0.01708 square 0.01358 least 0.01305 classifier 0.01151 least_square 0.01000 linear 0.00991 kernel 0.00832 support_vector_machine 0.00780 TOPIC 39 error 0.03970 packet 0.02570 layer 0.02393 cache 0.02024 performance 0.01413 loss 0.01286 rate 0.01261 tcp 0.01233 rfid 0.00951 buffer 0.00904 protocol 0.00840 congestion 0.00839 correction 0.00781 tag 0.00727 latency 0.00687 TOPIC 40 logic 0.02906 language 0.01847 tree 0.01795 theory 0.01510 type 0.01445 proof 0.01152 structure 0.00881 science 0.00788 order 0.00784 operator 0.00781 calculus 0.00750 computer 0.00719 computer_science 0.00668 reasoning 0.00658 grammar 0.00652 TOPIC 41 card 0.02214 cube 0.02188 password 0.02056 multicore 0.01593 cd 0.01159 clone 0.01058 phenotype 0.00937 weakly 0.00839 fp 0.00670 smart_card 0.00621 method 0.00547 authentication_protocol 0.00482 genotype 0.00460 premature 0.00396 shrinking 0.00381 TOPIC 42 antenna 0.04040 diversity 0.03721 mimo 0.02306 semi 0.01858 multiple 0.01535 beamforming 0.01179 pilot 0.01156 uwb 0.01136 performance 0.01014 transmit 0.00949 cooperative 0.00931 rayleigh 0.00900 label 0.00878 csi 0.00837 fading 0.00837 TOPIC 43 network 0.12054 node 0.04704 wireless 0.03001 protocol 0.02435 routing 0.02382 communication 0.01636 sensor 0.01564 ad 0.01386 hoc 0.01210 link 0.01037 wireless_sensor 0.00969 topology 0.00898 algorithm 0.00876 distributed 0.00832 ad_hoc 0.00828 TOPIC 44 method 0.03178 equation 0.02464 problem 0.02032 solution 0.01912 numerical 0.01551 function 0.01433 linear 0.01378 order 0.01171 convergence 0.00933 approximation 0.00926 differential 0.00832 convex 0.00825 nonlinear 0.00804 point 0.00768 boundary 0.00757 TOPIC 45 polynomial 0.02073 finite 0.01271 set 0.01270 function 0.01120 algebra 0.01111 number 0.01009 point 0.00922 theory 0.00894 class 0.00827 formula 0.00703 algebraic 0.00698 property 0.00685 group 0.00677 show 0.00660 one 0.00654 TOPIC 46 design 0.07400 circuit 0.05298 cloud 0.02110 chip 0.01823 delay 0.01143 gate 0.01098 clock 0.01097 proposed 0.00975 technique 0.00893 aided 0.00863 high 0.00802 timing 0.00788 power 0.00763 computer_aided 0.00760 logic 0.00746 TOPIC 47 channel 0.06366 communication 0.02124 performance 0.01685 scheme 0.01547 receiver 0.01504 proposed 0.01292 multiple 0.01092 transmission 0.01091 frequency 0.01035 relay 0.01026 signal 0.00974 wireless 0.00971 capacity 0.00884 power 0.00864 rate 0.00857 TOPIC 48 test 0.10866 fault 0.05912 testing 0.04815 generation 0.01833 diagnosis 0.01242 defect 0.01233 tolerance 0.01203 method 0.01155 technique 0.01139 case 0.01080 approach 0.00884 detection 0.00876 coverage 0.00808 fault_tolerance 0.00702 used 0.00628 TOPIC 49 decision 0.06791 making 0.01906 criterion 0.01700 set 0.01698 value 0.01480 uncertainty 0.01394 decision_making 0.01388 attribute 0.01361 information 0.00994 model 0.00953 problem 0.00860 preference 0.00839 approach 0.00821 theory 0.00804 method 0.00763 TOPIC 50 code 0.12302 kernel 0.03063 binary 0.02430 decoding 0.02011 equilibrium 0.01737 length 0.01221 information 0.01125 sequence 0.01080 decoder 0.00976 error 0.00899 ieee 0.00874 theory 0.00739 nash 0.00699 transaction 0.00687 coding 0.00673 TOPIC 51 eeg 0.02173 sentence 0.02028 relation 0.01909 journal 0.01754 symbolic 0.01739 offset 0.01660 computational 0.01293 compound 0.01245 shift 0.01204 chemistry 0.01105 summarization 0.01102 molecular 0.01073 atom 0.01030 android 0.00948 molecule 0.00878 TOPIC 52 program 0.04225 specification 0.01918 model 0.01515 constraint 0.01489 state 0.01391 language 0.01384 semantics 0.01091 property 0.01035 checking 0.00997 abstraction 0.00911 programming 0.00891 formal 0.00846 approach 0.00828 analysis 0.00797 action 0.00792 TOPIC 53 micro 0.03082 auction 0.02208 delta 0.01686 mechanism 0.01255 newton 0.01230 sigma 0.01186 vessel 0.01121 composite 0.01071 barrier 0.01000 reputation 0.00997 manufacturing 0.00883 acm_symposium 0.00879 acm 0.00862 penalty 0.00850 nano 0.00774 TOPIC 54 matrix 0.07336 sparse 0.02181 rank 0.01958 subspace 0.01874 algorithm 0.01765 column 0.01008 dictionary 0.00943 point 0.00835 iterative 0.00794 hyperspectral 0.00756 regularization 0.00734 correspondence 0.00719 linear 0.00711 row 0.00704 projection 0.00689 TOPIC 55 mobile 0.09356 location 0.04285 context 0.04183 user 0.03456 trust 0.02469 aware 0.01866 device 0.01456 phone 0.01453 mobile_device 0.01344 privacy 0.01317 information 0.01187 association 0.01181 application 0.00976 contract 0.00785 mobile_phone 0.00734 TOPIC 56 template 0.03468 spl 0.02430 descriptor 0.01710 coloring 0.01689 biometric 0.01593 sup 0.01429 skin 0.01410 sketch 0.01348 moment 0.01176 iris 0.01166 counting 0.01122 invariant 0.01098 bipartite 0.01055 rotation 0.00942 fi 0.00912 TOPIC 57 motion 0.04314 camera 0.02426 method 0.01746 position 0.01043 moving 0.01042 estimation 0.00993 trajectory 0.00993 image 0.00988 point 0.00897 calibration 0.00878 proposed 0.00826 human 0.00703 vision 0.00693 sar 0.00685 orientation 0.00678 TOPIC 58 control 0.10366 controller 0.02678 feedback 0.01547 dynamic 0.01436 proposed 0.01422 time 0.01308 nonlinear 0.01179 loop 0.01173 output 0.01150 adaptive 0.01134 design 0.01088 input 0.01067 state 0.00965 parameter 0.00913 delay 0.00876 TOPIC 59 technology 0.02199 application 0.02045 computing 0.02015 device 0.01743 environment 0.01334 internet 0.01327 service 0.01320 communication 0.01264 user 0.01031 infrastructure 0.00919 management 0.00913 challenge 0.00851 new 0.00818 information 0.00789 access 0.00775 TOPIC 60 network 0.03154 energy 0.02898 traffic 0.01901 performance 0.01379 load 0.01345 qos 0.01271 resource 0.01263 service 0.01258 scheme 0.00990 bandwidth 0.00952 scheduling 0.00930 consumption 0.00813 access 0.00795 delay 0.00772 proposed 0.00765 TOPIC 61 sentiment 0.01937 virtualization 0.01743 multidimensional 0.01676 erp 0.01581 convolutional 0.01514 model 0.01048 o 0.00896 deep 0.00886 convolutional_neural 0.00769 convolutional_neural_network 0.00726 analysis 0.00656 win 0.00612 augmentation 0.00601 consolidation 0.00598 olap 0.00578 TOPIC 62 query 0.10053 index 0.04272 spatial 0.04157 tree 0.03534 stream 0.02704 multicast 0.02257 correlation 0.01394 consensus 0.01291 data 0.00928 processing 0.00886 database 0.00876 voting 0.00817 structure 0.00816 indexing 0.00793 efficient 0.00788 TOPIC 63 model 0.05237 temperature 0.02338 thermal 0.01750 air 0.01306 plant 0.01143 water 0.01134 forecasting 0.01121 environmental 0.01117 predictive 0.00853 forecast 0.00754 process 0.00698 used 0.00692 prediction 0.00675 package 0.00671 steady 0.00643 TOPIC 64 gene 0.02558 protein 0.02093 sequence 0.01733 cell 0.01648 bioinformatics 0.01249 biological 0.01140 analysis 0.01017 expression 0.00974 biology 0.00901 method 0.00900 structure 0.00850 data 0.00811 genome 0.00761 dna 0.00732 cancer 0.00705 TOPIC 65 co 0.04279 satellite 0.03295 music 0.03008 land 0.01563 soil 0.01445 film 0.01283 international_geoscience 0.01065 musical 0.00938 microwave 0.00808 earth 0.00751 arabic 0.00696 concentration 0.00675 use 0.00637 occurrence 0.00633 thin 0.00600 TOPIC 66 algorithm 0.07724 genetic 0.05474 genetic_algorithm 0.03289 population 0.02255 placement 0.01694 ga 0.01421 evolutionary 0.01123 proposed 0.00977 problem 0.00940 method 0.00933 optimisation 0.00852 gas 0.00851 search 0.00745 programming 0.00722 fitness 0.00669 TOPIC 67 digital 0.03678 library 0.02445 information 0.01664 collection 0.01258 document 0.01240 content 0.00924 research 0.00842 book 0.00795 list 0.00773 metadata 0.00741 format 0.00670 scientific 0.00633 article 0.00625 digital_library 0.00623 ir 0.00620 TOPIC 68 optical 0.03733 body 0.01735 switching 0.01548 wavelength 0.01152 passive 0.01055 walking 0.01024 pulse 0.00952 fiber 0.00847 speed 0.00779 switch 0.00738 burst 0.00695 gait 0.00675 conversion 0.00671 digital 0.00613 blocking 0.00589 TOPIC 69 automaton 0.02378 box 0.01101 deterministic 0.01014 block 0.00997 round 0.00945 regular 0.00931 function 0.00923 complexity 0.00877 free 0.00825 one 0.00764 primitive 0.00705 input 0.00701 key 0.00688 bit 0.00677 string 0.00649 TOPIC 70 de 0.15040 la 0.03534 le 0.03161 social_medium 0.02090 en 0.02012 un 0.01987 et 0.01890 se 0.01382 une 0.01063 ce 0.01061 par 0.01043 pour 0.00960 el 0.00948 est 0.00791 dans 0.00770 TOPIC 71 model 0.03652 distribution 0.03087 probability 0.02154 estimation 0.01701 parameter 0.01650 random 0.01545 estimate 0.01223 statistical 0.01033 method 0.00993 function 0.00982 bayesian 0.00948 variable 0.00912 analysis 0.00877 probabilistic 0.00854 sampling 0.00813 TOPIC 72 software 0.07202 tool 0.02471 component 0.01832 application 0.01711 engineering 0.01415 language 0.01359 development 0.01286 programming 0.01271 design 0.01072 software_engineering 0.01044 code 0.00890 support 0.00765 implementation 0.00738 java 0.00736 approach 0.00702 TOPIC 73 failure 0.01926 response 0.01873 reliability 0.01821 brain 0.01592 effect 0.01477 study 0.01185 change 0.01073 subject 0.00946 long 0.00878 stress 0.00793 term 0.00694 stimulus 0.00651 analysis 0.00621 heart 0.00596 age 0.00567 TOPIC 74 network 0.10674 neural 0.07711 neural_network 0.05156 function 0.02973 net 0.02378 cost 0.01139 petri 0.00932 petri_net 0.00749 inventory 0.00733 model 0.00682 optimal 0.00601 problem 0.00593 weight 0.00527 layer 0.00512 algorithm 0.00504 TOPIC 75 robot 0.08970 localization 0.02110 robotics 0.02054 environment 0.01950 autonomous 0.01491 map 0.01259 navigation 0.01219 automation 0.01209 robotic 0.01159 mobile 0.01006 task 0.00987 planning 0.00953 approach 0.00765 robotics_automation 0.00710 international 0.00681 TOPIC 76 management 0.02032 process 0.01887 project 0.01854 business 0.01734 research 0.01584 development 0.01508 information 0.01405 technology 0.01188 it 0.01049 study 0.00957 organization 0.00945 product 0.00897 enterprise 0.00802 industry 0.00751 international 0.00734 TOPIC 77 service 0.07140 peer 0.04059 customer 0.02490 content 0.02474 provider 0.02100 message 0.01606 cloud 0.01512 cost 0.01475 peer_peer 0.01195 overlay 0.01112 service_provider 0.00937 maintenance 0.00918 replication 0.00871 delivery 0.00745 quality 0.00712 TOPIC 78 phase 0.05997 vehicle 0.05617 vehicular 0.02601 closed 0.01998 vehicular_technology 0.01552 fourier 0.01252 directional 0.01046 travel 0.00950 closed_form 0.00921 car 0.00916 closed_loop 0.00880 layout 0.00878 form 0.00838 transform 0.00832 time 0.00702 TOPIC 79 fuzzy 0.11379 rule 0.06980 particle 0.02680 proposed 0.01730 swarm 0.01668 method 0.01496 set 0.01322 algorithm 0.01307 man 0.01011 cybernetics 0.00987 pso 0.00936 particle_swarm 0.00919 man_cybernetics 0.00901 system_man_cybernetics 0.00900 fuzzy_logic 0.00879 TOPIC 80 parallel 0.04638 performance 0.02990 distributed 0.02278 processor 0.01584 application 0.01577 computing 0.01476 processing 0.01456 large 0.01081 high 0.01052 memory 0.01002 algorithm 0.00965 computation 0.00932 data 0.00918 execution 0.00916 implementation 0.00909 TOPIC 81 database 0.11273 view 0.06406 region 0.03800 relational 0.02393 assembly 0.02226 art 0.01438 embedding 0.01433 preserving 0.01309 state_art 0.01029 affine 0.01014 symmetry 0.00815 method 0.00771 relational_database 0.00753 point_view 0.00735 re 0.00676 TOPIC 82 power 0.07288 voltage 0.02068 low 0.01619 current 0.01586 frequency 0.01421 price 0.01062 supply 0.00987 high 0.00976 electronics 0.00957 market 0.00953 cmos 0.00831 dc 0.00805 circuit 0.00770 converter 0.00714 proposed 0.00658 TOPIC 83 object 0.17841 oriented 0.03654 object_oriented 0.01867 entity 0.01181 hierarchy 0.01161 schema 0.00879 class 0.00875 dependency 0.00834 representation 0.00771 relationship 0.00669 type 0.00605 approach 0.00566 information 0.00566 structure 0.00549 hierarchical 0.00515 TOPIC 84 clustering 0.04345 cluster 0.03570 distance 0.03242 similarity 0.02495 measure 0.02467 algorithm 0.02438 space 0.01977 method 0.01502 data 0.01286 curve 0.01039 dimensional 0.01021 set 0.00922 point 0.00886 neighbor 0.00792 mean 0.00743 TOPIC 85 search 0.06991 user 0.05957 web 0.02763 page 0.01890 engine 0.01841 information 0.01609 item 0.01345 recommendation 0.01336 ranking 0.00916 content 0.00867 search_engine 0.00825 site 0.00723 web_page 0.00698 searching 0.00660 approach 0.00643 TOPIC 86 policy 0.08777 access 0.03338 conflict 0.02580 control 0.01650 xml 0.01519 role 0.01042 standard 0.00750 iso 0.00643 authorization 0.00608 fish 0.00532 aes 0.00424 information 0.00420 regret 0.00411 iec 0.00392 model 0.00382 TOPIC 87 recovery 0.03641 false 0.03132 positive 0.02222 alarm 0.01102 lm 0.01028 theoretic 0.00959 false_positive 0.00959 affective 0.00958 game 0.00957 id 0.00889 or 0.00837 mc 0.00792 negative 0.00775 go 0.00769 false_alarm 0.00696 TOPIC 88 society 0.02325 personal 0.02267 copyright 0.02254 photo 0.02133 abstract 0.01906 spie 0.01600 japanese 0.01326 use 0.01249 permitted 0.01059 downloading 0.01039 tagging 0.01023 engineering 0.00873 duplicate 0.00870 ltd 0.00815 optical 0.00812 TOPIC 89 tolerant 0.02818 tensor 0.02192 fault_tolerant 0.02124 subsystem 0.01620 virtual_machine 0.01551 machine 0.00967 triple 0.00801 peer 0.00799 form 0.00697 completion 0.00685 totally 0.00637 liver 0.00625 refactoring 0.00624 order 0.00618 fault 0.00605 TOPIC 90 optimization 0.06506 algorithm 0.03056 objective 0.03047 problem 0.02803 evolutionary 0.01958 multi 0.01883 solution 0.01622 search 0.01486 strategy 0.01362 function 0.01008 approach 0.00911 global 0.00875 performance 0.00868 optimal 0.00850 secondary 0.00841 TOPIC 91 method 0.02981 feature 0.02752 classification 0.02086 selection 0.01760 algorithm 0.01607 recognition 0.01548 classifier 0.01483 proposed 0.01217 approach 0.01203 accuracy 0.01149 data 0.01136 set 0.01105 performance 0.01064 prediction 0.00973 series 0.00905 TOPIC 92 pattern 0.18201 matching 0.05482 sequential 0.01592 sequence 0.01419 frequent 0.01099 ml 0.00934 algorithm 0.00815 match 0.00656 morphological 0.00629 method 0.00576 plan 0.00546 drift 0.00539 structure 0.00507 rao 0.00477 af 0.00454 TOPIC 93 path 0.11721 lattice 0.01911 shortest 0.01625 shortest_path 0.01169 algorithm 0.00874 lock 0.00801 deadlock 0.00771 math 0.00731 pointer 0.00595 free 0.00590 bond 0.00582 grain 0.00555 method 0.00526 synchronization 0.00498 race 0.00452 TOPIC 94 sound 0.03892 transaction 0.02633 ieice 0.02483 contact 0.02458 ieice_transaction 0.02194 anti 0.01529 auditory 0.01452 seed 0.00951 bio 0.00938 digit 0.00783 bilateral 0.00742 native 0.00704 crop 0.00689 communication 0.00684 fundamental 0.00671 TOPIC 95 text 0.02740 language 0.02626 word 0.02531 document 0.02417 retrieval 0.02286 information 0.01528 approach 0.01070 model 0.01056 method 0.01012 corpus 0.00888 semantic 0.00871 task 0.00806 topic 0.00796 translation 0.00786 annotation 0.00698 TOPIC 96 scheme 0.09094 key 0.03849 group 0.03510 signature 0.02440 secure 0.01441 security 0.01202 encryption 0.01194 secret 0.01102 protocol 0.01071 identity 0.01002 proposed 0.00942 message 0.00860 public 0.00859 efficient 0.00793 hash 0.00768 TOPIC 97 fluid 0.01460 method 0.01393 flow 0.01310 physic 0.01102 simulation 0.01023 chemical 0.00965 reaction 0.00918 heat 0.00786 velocity 0.00714 algorithm 0.00676 computational 0.00619 solid 0.00579 two 0.00565 gaze 0.00560 computer 0.00506 TOPIC 98 force 0.04616 manipulator 0.02108 robot 0.01930 collision 0.01856 der 0.01643 arm 0.01592 obstacle 0.01575 bank 0.01557 und 0.01516 die 0.01495 sign 0.01365 joint 0.01247 intelligent_robot 0.01235 von 0.01120 intelligent_robot_system 0.01098 TOPIC 99 patient 0.03669 medical 0.02571 clinical 0.01691 care 0.01313 health 0.01283 assessment 0.01245 study 0.01004 treatment 0.00886 informatics 0.00852 hospital 0.00793 quality 0.00785 biomedical 0.00641 information 0.00597 protein 0.00584 medicine 0.00568
Visualisation¶
(this takes way too long)
In [18]:
vis_data = gensimvis.prepare(
tmdblp,
corpus=corpus_dblp,
dictionary=dict_dblp,
n_jobs=30,
sort_topics=False,
mds='mmds')
In [ ]:
# save in html for viewing
# pyLDAvis.save_html(vis_data, open('../src/tmdblp/res/ldavis.html', 'w'))
Word cloud¶
In [ ]:
for i in range(tmdblp.num_topics):
terms = tmdblp.get_topic_terms(i, topn=15)
wordcloud_dict = {}
for (a, b) in terms:
wordcloud_dict[dict_dblp[a]] = b
wordcloud = WordCloud(
mode='RGB',
relative_scaling=0.,
background_color='white',
colormap='tab20').generate_from_frequencies(wordcloud_dict)
wordcloud.to_file('./topicModel/src/tmdblp/res/topic{}.png'.format(i))
Misc¶
Compare c_v score:¶
In [83]:
def get_probs(tm,dictionary,corpus,text,coherence="c_v",topn=15):
"""
Gather the probability for the top_topics
"""
score = tm.top_topics(dictionary=dictionary,corpus=corpus,texts=text,coherence=coherence,topn=topn)
probs = []
for (a,b) in score:
probs.append(b)
mean = np.mean(probs)
median = np.median(probs)
return probs, mean,median
In [112]:
tmrest_meta_10 = models.LdaModel.load('../src/tmrest_meta/ldamodel_nb_topics_10')
tmfull_meta_10 = models.LdaModel.load('../src/tmfull_meta/ldamodel_nb_topics_10')
tmrest_meta_cv_10 = get_probs(tmrest_meta_10, dict_tmrest_meta, corpus_tmrest_meta, text_tmrest_meta)
tmfull_meta_cv_10 = get_probs(tmfull_meta_10, dict_tmfull_meta, corpus_tmfull_meta, text_tmfull_meta)
tmrest_meta_25 = models.LdaModel.load('../src/tmrest_meta/ldamodel_nb_topics_25')
tmfull_meta_25 = models.LdaModel.load('../src/tmfull_meta/ldamodel_nb_topics_25')
tmrest_meta_cv_25 = get_probs(tmrest_meta_25, dict_tmrest_meta, corpus_tmrest_meta, text_tmrest_meta)
tmfull_meta_cv_25 = get_probs(tmfull_meta_25, dict_tmfull_meta, corpus_tmfull_meta, text_tmfull_meta)
tmrest_meta_30 = models.LdaModel.load('../src/tmrest_meta/ldamodel_nb_topics_30')
tmfull_meta_30 = models.LdaModel.load('../src/tmfull_meta/ldamodel_nb_topics_30')
tmrest_meta_cv_30 = get_probs(tmrest_meta_30, dict_tmrest_meta, corpus_tmrest_meta, text_tmrest_meta)
tmfull_meta_cv_30 = get_probs(tmfull_meta_30, dict_tmfull_meta, corpus_tmfull_meta, text_tmfull_meta)
tmrest_meta_40 = models.LdaModel.load('../src/tmrest_meta/ldamodel_nb_topics_40')
tmfull_meta_40 = models.LdaModel.load('../src/tmfull_meta/ldamodel_nb_topics_40')
tmrest_meta_cv_40 = get_probs(tmrest_meta_40, dict_tmrest_meta, corpus_tmrest_meta, text_tmrest_meta)
tmfull_meta_cv_40 = get_probs(tmfull_meta_40, dict_tmfull_meta, corpus_tmfull_meta, text_tmfull_meta)
2018-03-29 00:55:03,043 : INFO : loading LdaModel object from ../src/tmrest_meta/ldamodel_nb_topics_10 2018-03-29 00:55:03,327 : INFO : loading expElogbeta from ../src/tmrest_meta/ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-29 00:55:03,337 : INFO : setting ignored attribute id2word to None 2018-03-29 00:55:03,338 : INFO : setting ignored attribute dispatcher to None 2018-03-29 00:55:03,339 : INFO : setting ignored attribute state to None 2018-03-29 00:55:03,339 : INFO : loaded ../src/tmrest_meta/ldamodel_nb_topics_10 2018-03-29 00:55:03,340 : INFO : loading LdaModel object from ../src/tmrest_meta/ldamodel_nb_topics_10.state 2018-03-29 00:55:03,349 : INFO : loaded ../src/tmrest_meta/ldamodel_nb_topics_10.state 2018-03-29 00:55:03,356 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_10 2018-03-29 00:55:03,376 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_10.expElogbeta.npy with mmap=None 2018-03-29 00:55:03,381 : INFO : setting ignored attribute dispatcher to None 2018-03-29 00:55:03,381 : INFO : setting ignored attribute state to None 2018-03-29 00:55:03,382 : INFO : setting ignored attribute id2word to None 2018-03-29 00:55:03,382 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_10 2018-03-29 00:55:03,383 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_10.state 2018-03-29 00:55:03,406 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_10.state 2018-03-29 00:55:03,429 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-29 00:55:04,571 : INFO : 43 batches submitted to accumulate stats from 2752 documents (-37317 virtual) 2018-03-29 00:55:05,042 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:05,048 : INFO : accumulator serialized 2018-03-29 00:55:05,084 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:05,091 : INFO : accumulator serialized 2018-03-29 00:55:05,124 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:05,130 : INFO : accumulator serialized 2018-03-29 00:55:05,160 : INFO : 3 accumulators retrieved from output queue 2018-03-29 00:55:05,196 : INFO : accumulated word occurrence stats for 48376 virtual documents 2018-03-29 00:55:05,527 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-29 00:55:08,792 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:08,799 : INFO : accumulator serialized 2018-03-29 00:55:08,844 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:08,849 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:08,856 : INFO : accumulator serialized 2018-03-29 00:55:08,851 : INFO : accumulator serialized 2018-03-29 00:55:08,903 : INFO : 3 accumulators retrieved from output queue 2018-03-29 00:55:08,943 : INFO : accumulated word occurrence stats for 91588 virtual documents 2018-03-29 00:55:09,258 : INFO : loading LdaModel object from ../src/tmrest_meta/ldamodel_nb_topics_25 2018-03-29 00:55:09,259 : INFO : loading expElogbeta from ../src/tmrest_meta/ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-29 00:55:09,262 : INFO : setting ignored attribute id2word to None 2018-03-29 00:55:09,263 : INFO : setting ignored attribute dispatcher to None 2018-03-29 00:55:09,263 : INFO : setting ignored attribute state to None 2018-03-29 00:55:09,264 : INFO : loaded ../src/tmrest_meta/ldamodel_nb_topics_25 2018-03-29 00:55:09,264 : INFO : loading LdaModel object from ../src/tmrest_meta/ldamodel_nb_topics_25.state 2018-03-29 00:55:09,273 : INFO : loaded ../src/tmrest_meta/ldamodel_nb_topics_25.state 2018-03-29 00:55:09,277 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_25 2018-03-29 00:55:09,279 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_25.expElogbeta.npy with mmap=None 2018-03-29 00:55:09,282 : INFO : setting ignored attribute dispatcher to None 2018-03-29 00:55:09,283 : INFO : setting ignored attribute state to None 2018-03-29 00:55:09,283 : INFO : setting ignored attribute id2word to None 2018-03-29 00:55:09,284 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_25 2018-03-29 00:55:09,285 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_25.state 2018-03-29 00:55:09,296 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_25.state 2018-03-29 00:55:09,306 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-29 00:55:12,979 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:12,994 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:13,008 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:12,985 : INFO : accumulator serialized 2018-03-29 00:55:13,001 : INFO : accumulator serialized 2018-03-29 00:55:13,015 : INFO : accumulator serialized 2018-03-29 00:55:13,149 : INFO : 3 accumulators retrieved from output queue 2018-03-29 00:55:13,300 : INFO : accumulated word occurrence stats for 48403 virtual documents 2018-03-29 00:55:14,120 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-29 00:55:22,346 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:22,352 : INFO : accumulator serialized 2018-03-29 00:55:22,396 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:22,477 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:22,403 : INFO : accumulator serialized 2018-03-29 00:55:22,485 : INFO : accumulator serialized 2018-03-29 00:55:22,605 : INFO : 3 accumulators retrieved from output queue 2018-03-29 00:55:22,772 : INFO : accumulated word occurrence stats for 91652 virtual documents 2018-03-29 00:55:23,593 : INFO : loading LdaModel object from ../src/tmrest_meta/ldamodel_nb_topics_30 2018-03-29 00:55:23,595 : INFO : loading expElogbeta from ../src/tmrest_meta/ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-29 00:55:23,597 : INFO : setting ignored attribute id2word to None 2018-03-29 00:55:23,598 : INFO : setting ignored attribute dispatcher to None 2018-03-29 00:55:23,598 : INFO : setting ignored attribute state to None 2018-03-29 00:55:23,599 : INFO : loaded ../src/tmrest_meta/ldamodel_nb_topics_30 2018-03-29 00:55:23,599 : INFO : loading LdaModel object from ../src/tmrest_meta/ldamodel_nb_topics_30.state 2018-03-29 00:55:23,609 : INFO : loaded ../src/tmrest_meta/ldamodel_nb_topics_30.state 2018-03-29 00:55:23,613 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_30 2018-03-29 00:55:23,614 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_30.expElogbeta.npy with mmap=None 2018-03-29 00:55:23,617 : INFO : setting ignored attribute dispatcher to None 2018-03-29 00:55:23,618 : INFO : setting ignored attribute state to None 2018-03-29 00:55:23,618 : INFO : setting ignored attribute id2word to None 2018-03-29 00:55:23,619 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_30 2018-03-29 00:55:23,619 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_30.state 2018-03-29 00:55:23,633 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_30.state 2018-03-29 00:55:23,642 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-29 00:55:27,894 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:27,928 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:27,900 : INFO : accumulator serialized 2018-03-29 00:55:27,934 : INFO : accumulator serialized 2018-03-29 00:55:28,102 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:28,108 : INFO : accumulator serialized 2018-03-29 00:55:28,225 : INFO : 3 accumulators retrieved from output queue 2018-03-29 00:55:28,419 : INFO : accumulated word occurrence stats for 48403 virtual documents 2018-03-29 00:55:29,418 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-29 00:55:38,713 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:38,825 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:38,720 : INFO : accumulator serialized 2018-03-29 00:55:38,874 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:38,832 : INFO : accumulator serialized 2018-03-29 00:55:38,882 : INFO : accumulator serialized 2018-03-29 00:55:39,034 : INFO : 3 accumulators retrieved from output queue 2018-03-29 00:55:39,263 : INFO : accumulated word occurrence stats for 91725 virtual documents 2018-03-29 00:55:40,282 : INFO : loading LdaModel object from ../src/tmrest_meta/ldamodel_nb_topics_40 2018-03-29 00:55:40,324 : INFO : loading expElogbeta from ../src/tmrest_meta/ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-29 00:55:40,340 : INFO : setting ignored attribute id2word to None 2018-03-29 00:55:40,341 : INFO : setting ignored attribute dispatcher to None 2018-03-29 00:55:40,342 : INFO : setting ignored attribute state to None 2018-03-29 00:55:40,342 : INFO : loaded ../src/tmrest_meta/ldamodel_nb_topics_40 2018-03-29 00:55:40,343 : INFO : loading LdaModel object from ../src/tmrest_meta/ldamodel_nb_topics_40.state 2018-03-29 00:55:40,381 : INFO : loaded ../src/tmrest_meta/ldamodel_nb_topics_40.state 2018-03-29 00:55:40,384 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_40 2018-03-29 00:55:40,395 : INFO : loading expElogbeta from ../src/tmfull_meta/ldamodel_nb_topics_40.expElogbeta.npy with mmap=None 2018-03-29 00:55:40,434 : INFO : setting ignored attribute dispatcher to None 2018-03-29 00:55:40,435 : INFO : setting ignored attribute state to None 2018-03-29 00:55:40,435 : INFO : setting ignored attribute id2word to None 2018-03-29 00:55:40,435 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_40 2018-03-29 00:55:40,436 : INFO : loading LdaModel object from ../src/tmfull_meta/ldamodel_nb_topics_40.state 2018-03-29 00:55:40,504 : INFO : loaded ../src/tmfull_meta/ldamodel_nb_topics_40.state 2018-03-29 00:55:40,520 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-29 00:55:46,075 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:46,122 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:46,158 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:46,080 : INFO : accumulator serialized 2018-03-29 00:55:46,129 : INFO : accumulator serialized 2018-03-29 00:55:46,164 : INFO : accumulator serialized 2018-03-29 00:55:46,375 : INFO : 3 accumulators retrieved from output queue 2018-03-29 00:55:46,672 : INFO : accumulated word occurrence stats for 48487 virtual documents 2018-03-29 00:55:48,021 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-03-29 00:55:59,757 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:59,769 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:59,763 : INFO : accumulator serialized 2018-03-29 00:55:59,776 : INFO : accumulator serialized 2018-03-29 00:55:59,970 : INFO : serializing accumulator to return to master... 2018-03-29 00:55:59,978 : INFO : accumulator serialized 2018-03-29 00:56:00,188 : INFO : 3 accumulators retrieved from output queue 2018-03-29 00:56:00,538 : INFO : accumulated word occurrence stats for 91739 virtual documents
In [120]:
print(tmrest_meta_cv_10[1],tmrest_meta_cv_25[1],tmrest_meta_cv_30[1],tmrest_meta_cv_40[1])
0.5017054480283049 0.4903675425067642 0.5082615420437593 0.49662381709137027
In [68]:
tmfull_meta_score = pd.read_csv('../src/tmfull_meta/scores.combined.csv')
tmrest_meta_score = pd.read_csv('../src/tmrest_meta/scores.combined.csv')
In [118]:
f = plt.figure(figsize=(8, 8))
ax1 = f.add_subplot(211)
xs = tmfull_meta_score.num_topic.tolist()
l11 = ax1.plot(xs, tmfull_meta_score.c_v.tolist(), color='b', linestyle='-')
l21 = ax1.plot(xs, tmrest_meta_score.c_v.tolist(), color='r', linestyle='-')
ax1.set_ylabel('$C_V$ coherence score') #, color='b')
best = tmfull_meta_score.c_v.argmax()
ax1.scatter(
x=tmfull_meta_score.num_topic.iloc[best],
y=tmfull_meta_score.iloc[best].c_v,
c='g',
marker='*',
s=150)
best = tmrest_meta_score.c_v.argmax()
ax1.scatter(
x=tmrest_meta_score.num_topic.iloc[best],
y=tmrest_meta_score.iloc[best].c_v,
c='g',
marker='*',
s=150)
# ax2 = ax1.twinx()
# l12 = ax2.plot(
# xs, tmfull_meta_score.u_mass.tolist(), color='m', linestyle='-')
# l22 = ax2.plot(
# xs, tmrest_meta_score.u_mass.tolist(), color='m', linestyle='-.')
# ax2.set_ylabel('u_mass coherence score', color='m')
ax1.legend(
l11 + l21, #+l12+l22,
['tm-20yr', 'tm-6yr'], #, 'u_mass (tm-20yr)', 'u_mass (tm-6yr)'],
loc='upper right')
# title='Coherence Score',
# loc='upper center',
# bbox_to_anchor=(0.5, 1.25),
# ncol=2,
# fancybox=True)
ax1.set_xlabel('Number of topics, $k$')
ax2 = f.add_subplot(223)
sns.distplot(tmfull_meta_cv_10[0], ax=ax2, hist=False)
sns.distplot(tmfull_meta_cv_25[0], ax=ax2, hist=False)
sns.distplot(tmfull_meta_cv_30[0], ax=ax2, hist=False)
sns.distplot(tmfull_meta_cv_40[0], ax=ax2, hist=False)
ax2.set_title('tm-20yr')
ax2.set_xlim(0, 1)
ax2.set_xlabel('$C_V$ coherence score')
ax2.set_ylabel('Distribution')
ax3 = f.add_subplot(224, sharey=ax2)
sns.distplot(tmrest_meta_cv_10[0], ax=ax3, hist=False, label='10')
sns.distplot(tmrest_meta_cv_25[0], ax=ax3, hist=False, label='25')
sns.distplot(tmrest_meta_cv_30[0], ax=ax3, hist=False, label='30')
sns.distplot(tmrest_meta_cv_40[0], ax=ax3, hist=False, label='40')
ax3.set_title('tm-6yr')
ax3.set_xlabel('$C_V$ coherence score')
ax3.set_xlim(0, 1)
ax3.legend(loc=0)
f.tight_layout()
/Users/weiting/miniconda3/envs/infnet3/lib/python3.6/site-packages/statsmodels/nonparametric/kde.py:475: DeprecationWarning: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer. grid,delta = np.linspace(a,b,gridsize,retstep=True) /Users/weiting/miniconda3/envs/infnet3/lib/python3.6/site-packages/statsmodels/nonparametric/kde.py:475: DeprecationWarning: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer. grid,delta = np.linspace(a,b,gridsize,retstep=True) /Users/weiting/miniconda3/envs/infnet3/lib/python3.6/site-packages/statsmodels/nonparametric/kde.py:475: DeprecationWarning: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer. grid,delta = np.linspace(a,b,gridsize,retstep=True) /Users/weiting/miniconda3/envs/infnet3/lib/python3.6/site-packages/statsmodels/nonparametric/kde.py:475: DeprecationWarning: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer. grid,delta = np.linspace(a,b,gridsize,retstep=True) /Users/weiting/miniconda3/envs/infnet3/lib/python3.6/site-packages/statsmodels/nonparametric/kde.py:475: DeprecationWarning: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer. grid,delta = np.linspace(a,b,gridsize,retstep=True) /Users/weiting/miniconda3/envs/infnet3/lib/python3.6/site-packages/statsmodels/nonparametric/kde.py:475: DeprecationWarning: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer. grid,delta = np.linspace(a,b,gridsize,retstep=True) /Users/weiting/miniconda3/envs/infnet3/lib/python3.6/site-packages/statsmodels/nonparametric/kde.py:475: DeprecationWarning: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer. grid,delta = np.linspace(a,b,gridsize,retstep=True) /Users/weiting/miniconda3/envs/infnet3/lib/python3.6/site-packages/statsmodels/nonparametric/kde.py:475: DeprecationWarning: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer. grid,delta = np.linspace(a,b,gridsize,retstep=True)
In [121]:
f.savefig('./IMG/coherence_scores.png', format='png', bbox_inches='tight')
ARCHIVE BELOW:¶
Model 1: Full Series of Publications¶
From preprocess_pubs.ipynb in infnet-scrapper, we generate tokens for our publications.
Now, we use these tokens to create a dictionary, corpus, and topic models for each of the documents
In [9]:
combine_toks = lambda row: row.summary_toks + row.bigram_toks + row.trigram_toks
In [10]:
pub_toks['combined_toks'] = pub_toks.apply(combine_toks, axis=1)
In [11]:
print((pub_toks.iloc[0].combined_toks))
['ballooning', 'stability', 'analysis', 'jet', 'hmode', 'discharge', 'previous', 'study', 'stability', 'large', 'aspect', 'ratio', 'model', 'equilibrium', 'ideal', 'mhd', 'ballooning', 'mode', 'shown', 'across', 'bulk', 'plasma', 'exist', 'two', 'marginally', 'stable', 'value', 'pressure', 'gradient', 'parameter', 'define', 'unstable', 'zone', 'separate', 'first', 'small', 'stable', 'region', 'second', 'large', 'stable', 'region', 'close', 'separatrix', 'however', 'first', 'second', 'region', 'coalesce', 'surface', 'averaged', 'current', 'density', 'exceeds', 'critical', 'value', 'plasma', 'region', 'stable', 'ballooning', 'mode', 'value', 'pressure', 'gradient', 'paper', 'extend', 'result', 'jet', 'mode', 'equilibrium', 'using', 'finite', 'aspect', 'ratio', 'ballooning', 'formalism', 'ass', 'relevance', 'ideal', 'ballooning', 'stability', 'discharge', 'particular', 'analyse', 'shot', 'time', 'sec', 'phase', 'proceeding', '16th', 'european', 'conference', 'controlled', 'fusion', 'plasma', 'physic', 'european_conference']
In [12]:
docs = pub_toks.combined_toks.tolist()
# Create the dictionary mapping:
dictionary = Dictionary(docs)
# ignore words that appear in less than 10 documents or more than 50% documents
dictionary.filter_extremes(no_below=10, no_above=0.5)
dictionary.compactify()
# Create a bow tagging for each publication:
bowified = lambda row: dictionary.doc2bow(row.combined_toks)
pub_toks['bow'] = pub_toks.apply(bowified, axis=1)
# Generate a corpus based on the tokens, which we will be using later
corpus = pub_toks.bow.tolist()
2018-01-20 00:14:58,894 : INFO : adding document #0 to Dictionary(0 unique tokens: [])
2018-01-20 00:14:59,685 : INFO : built Dictionary(27911 unique tokens: ['ballooning', 'stability', 'analysis', 'jet', 'hmode']...) from 8867 documents (total 771590 corpus positions)
2018-01-20 00:14:59,727 : INFO : discarding 21452 tokens: [('ballooning', 9), ('jet', 8), ('hmode', 4), ('discharge', 8), ('mhd', 6), ('marginally', 5), ('zone', 9), ('separatrix', 5), ('coalesce', 2), ('sec', 9)]...
2018-01-20 00:14:59,728 : INFO : keeping 6459 tokens which were in no less than 10 and no more than 4433 (=50.0%) documents
2018-01-20 00:14:59,740 : INFO : resulting dictionary: Dictionary(6459 unique tokens: ['stability', 'analysis', 'previous', 'study', 'large']...)
In [13]:
print(('Number of unique tokens:', len(dictionary)))
print(('Number of documents: ', len(corpus)))
('Number of unique tokens:', 6459)
('Number of documents: ', 8867)
In [14]:
pub_toks.head(2)
Out[14]:
| year | title | abstract | publications | summary | summary_toks | bigram_toks | trigram_toks | combined_toks | bow | |
|---|---|---|---|---|---|---|---|---|---|---|
| pub_id | ||||||||||
| 400818dc-63af-4a26-80c5-906f98e1f8ab | 1989 | ballooning stability analysis of jet hmode dis... | previous studies of the stability of a large a... | proceedings 16th european conference on contr... | ballooning stability analysis of jet hmode dis... | [ballooning, stability, analysis, jet, hmode, ... | [european_conference] | [] | [ballooning, stability, analysis, jet, hmode, ... | [(0, 3), (1, 1), (2, 1), (3, 1), (4, 2), (5, 2... |
| 18b1a861-afef-4fff-bc80-d02e05be18c4 | 2013 | query processing in data integration | in this chapter we illustrate the main techniq... | data exchange, integration, and streams,dagstu... | query processing in data integration. in this ... | [query, processing, data, integration, chapter... | [query_answering, integrity_constraint, semist... | [] | [query, processing, data, integration, chapter... | [(7, 1), (25, 1), (28, 1), (58, 8), (59, 4), (... |
In [15]:
# Save the dictionary:
dictionary.save('../data/fullpub/dictionary_fullpub.pkl', pickle_protocol=-1)
# save the pub_toks:
pub_toks.to_pickle('../data/fullpub/pub_toks_wBOW_fullpub.pkl')
# save the corpus:
pkl.dump(corpus, open('../data/fullpub/corpus_fullpub.pkl', 'wb'), -1)
2018-01-20 00:15:09,085 : INFO : saving Dictionary object under ../data/fullpub/dictionary_fullpub.pkl, separately None 2018-01-20 00:15:09,089 : INFO : saved ../data/fullpub/dictionary_fullpub.pkl
LDA Modelling:¶
In [30]:
num_topics = 50
passes = 30 # total number of times the corpus is seen
iterations = 1000 # how many times each document is seen
chunksize = 2000 # how many documents each mini-batch
update_every = 0 # Batch learning
eval_every = None
In [25]:
tmp = dictionary[0]
id2word = dictionary.id2token
In [31]:
# This is the fullpub LDA model.
lda = models.ldamodel.LdaModel(
corpus=corpus,
id2word=id2word,
chunksize=chunksize,
eta='auto',
alpha='auto',
num_topics=num_topics,
iterations=iterations,
passes=passes,
update_every=update_every,
eval_every=eval_every,
random_state=rng)
2018-01-20 00:58:03,674 : INFO : using autotuned alpha, starting with [0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02, 0.02] 2018-01-20 00:58:03,678 : INFO : using serial LDA version on this node 2018-01-20 00:58:05,807 : INFO : running batch LDA training, 50 topics, 30 passes over the supplied corpus of 8867 documents, updating model once every 8867 documents, evaluating perplexity every 0 documents, iterating 1000x with a convergence threshold of 0.001000 2018-01-20 00:58:05,808 : INFO : PROGRESS: pass 0, at document #2000/8867 2018-01-20 00:58:31,693 : INFO : optimized alpha [0.018586102399558112, 0.0187478064484662, 0.018182244123782196, 0.019022041470034706, 0.018431945813717533, 0.018652079237067062, 0.018422598341878916, 0.018621231415384718, 0.01836679214189393, 0.018761278261322529, 0.018542770546325891, 0.018663214094428421, 0.018389930688392581, 0.018246230998905752, 0.0183014093333763, 0.018950294584420595, 0.01857394495797755, 0.018937407261583484, 0.01855249413239006, 0.018301378180923675, 0.018509111230388946, 0.018638757743251688, 0.01875013607564719, 0.018410804280102257, 0.018387826648123582, 0.018553653270080253, 0.018895070145477425, 0.018707042397130055, 0.018299057909394935, 0.018247457588949561, 0.018671605957674278, 0.018749224670180843, 0.018193593746106591, 0.018924839418235279, 0.018236060739011357, 0.018378649955478159, 0.018388413009000893, 0.018902333387164807, 0.018564064330674188, 0.01857358326157035, 0.018400618444226774, 0.018322808591899383, 0.018640690557452082, 0.018344680020956197, 0.019201564164934133, 0.018408237916104703, 0.018715081160008768, 0.018335223190050199, 0.018290530937791098, 0.018476662072716945] 2018-01-20 00:58:31,694 : INFO : PROGRESS: pass 0, at document #4000/8867 2018-01-20 00:58:56,842 : INFO : optimized alpha [0.017456746100059803, 0.017544559446785554, 0.016727566181799824, 0.01805244498739212, 0.016935602698207888, 0.017239158363150608, 0.017067511480067039, 0.01742846573736152, 0.016901890354045521, 0.017771267610646083, 0.017420780563111131, 0.017431853035313308, 0.017100961209799734, 0.016682824828226764, 0.016856901353370563, 0.01799323487078109, 0.017255404266713955, 0.017983221210673757, 0.017266904669433092, 0.016798885248044841, 0.017241397898043399, 0.01729953315020118, 0.017648512869498589, 0.017168073880684386, 0.017029072505855569, 0.017360257081105324, 0.017586010781136318, 0.01738865354877555, 0.016815597143095832, 0.016753041160172767, 0.017743817123025196, 0.017657128743562747, 0.016826671316264752, 0.0179997112310104, 0.016881680772197875, 0.016941224461924899, 0.016990005957270352, 0.017994315785299416, 0.017449209219353379, 0.017325496360378407, 0.017000493297328136, 0.017104158828802316, 0.017392068846031266, 0.016932205252499694, 0.018534343581408892, 0.017115123093273319, 0.01744396730125683, 0.016835185317383747, 0.016828806105751527, 0.017140242854627556] 2018-01-20 00:58:56,843 : INFO : PROGRESS: pass 0, at document #6000/8867 2018-01-20 00:59:22,974 : INFO : optimized alpha [0.016571612708041788, 0.01642802378790708, 0.015398611984487449, 0.017201566429213808, 0.015726238327231471, 0.016039569385883044, 0.015930889589209313, 0.016312675935130843, 0.01567950317583491, 0.016998374698816435, 0.016334020571411461, 0.016408120068630131, 0.016031700946280612, 0.01538753895922579, 0.015596106169311629, 0.017152337829248215, 0.016109478500038424, 0.01717068186251431, 0.016239247968198468, 0.015611361764104504, 0.016031831520260845, 0.016211094984771583, 0.016470079268195634, 0.016061292326683525, 0.01585250498234379, 0.016282189677174892, 0.016606930383846735, 0.016400401095905667, 0.015517128529301084, 0.015528873945543836, 0.016866623800645464, 0.016686194503573216, 0.015580982157624263, 0.017048357166040383, 0.015726887195179194, 0.01577725522918148, 0.015736948596744786, 0.017065737679575452, 0.016311002178453344, 0.016269184388274259, 0.015883505825363731, 0.015953347150554676, 0.016337617181200366, 0.015814546661073403, 0.017709087235509843, 0.01587848774851679, 0.016210626864696907, 0.015643202245388968, 0.01559211026423683, 0.015890388157456185] 2018-01-20 00:59:22,975 : INFO : PROGRESS: pass 0, at document #8000/8867 2018-01-20 00:59:48,967 : INFO : optimized alpha [0.015728827438760144, 0.015628692300972746, 0.014334221582670521, 0.016543377915016651, 0.014696981705376255, 0.015076548733396748, 0.014854969747072821, 0.015399839216060177, 0.01454787681773354, 0.016243119324362957, 0.015303338491993187, 0.015480062122641171, 0.014966955600829614, 0.014299772420005843, 0.014593741194783001, 0.016453163490073214, 0.015025356395007412, 0.016422130115446361, 0.015275131950999454, 0.014507939200465756, 0.01503705444773752, 0.015215953564265186, 0.015482132556331072, 0.015104159328687685, 0.014882845509190572, 0.015409606043290128, 0.015786733933656219, 0.015404221964881554, 0.014394347178544133, 0.014421015170997423, 0.016066497497656047, 0.015810731708072154, 0.014498655259807907, 0.016168755849615822, 0.014646654581125526, 0.014665137820544061, 0.01460582132242751, 0.016137252243376556, 0.015353578372336567, 0.015230389162394836, 0.01487534573546841, 0.01488423841641868, 0.015348189614935658, 0.014764489539727683, 0.017369227297324361, 0.014902969838315771, 0.015291300165201161, 0.014568202058774179, 0.014433170771730069, 0.0148036580777929] 2018-01-20 00:59:48,968 : INFO : PROGRESS: pass 0, at document #8867/8867 2018-01-20 00:59:59,730 : INFO : optimized alpha [0.01503714485178553, 0.014813746304980752, 0.013419682545412303, 0.01581003093725377, 0.013640321803180708, 0.014185245513551407, 0.013865992146425402, 0.014559656366692328, 0.013621508736145049, 0.015550561806378349, 0.014362253694966683, 0.014725241611809624, 0.014035954727021306, 0.013336309292962532, 0.013606832784001337, 0.015630905229538323, 0.014104871507925693, 0.015481161387110465, 0.014488748121826369, 0.013515680471105372, 0.014282129643807385, 0.014381207720728262, 0.014687526712836468, 0.014116494472950851, 0.013944924752907274, 0.014586352399677142, 0.015146414289164464, 0.014580940645914022, 0.013365155835583288, 0.013599710097311573, 0.015135490056651839, 0.01514780293472319, 0.013560354928042525, 0.01540306703296339, 0.013777731186800545, 0.013630873824735457, 0.013688104091755565, 0.015435026909899865, 0.014424705978242748, 0.014318275414119366, 0.013994541180839171, 0.013982581498750694, 0.014475090131126684, 0.013824196990488811, 0.016700932877022675, 0.014070425786419828, 0.014408828339468072, 0.013619627213251351, 0.013468899719233474, 0.013839496690695898] 2018-01-20 01:00:00,937 : INFO : topic #28 (0.013): 0.009*"model" + 0.008*"system" + 0.006*"analysis" + 0.005*"set" + 0.005*"result" + 0.004*"learning" + 0.004*"used" + 0.004*"show" + 0.004*"paper" + 0.004*"whether" 2018-01-20 01:00:00,939 : INFO : topic #13 (0.013): 0.009*"proceeding" + 0.008*"computational" + 0.007*"system" + 0.007*"query" + 0.007*"conference" + 0.006*"theory" + 0.006*"algorithm" + 0.006*"learning" + 0.006*"measure" + 0.006*"result" 2018-01-20 01:00:00,940 : INFO : topic #15 (0.016): 0.011*"system" + 0.009*"model" + 0.008*"data" + 0.006*"method" + 0.006*"conference" + 0.005*"performance" + 0.005*"approach" + 0.005*"word" + 0.005*"paper" + 0.005*"using" 2018-01-20 01:00:00,941 : INFO : topic #3 (0.016): 0.018*"model" + 0.012*"system" + 0.009*"based" + 0.008*"translation" + 0.008*"language" + 0.007*"method" + 0.007*"data" + 0.007*"proceeding" + 0.005*"conference" + 0.005*"computational" 2018-01-20 01:00:00,942 : INFO : topic #44 (0.017): 0.014*"model" + 0.010*"system" + 0.008*"language" + 0.007*"data" + 0.007*"using" + 0.006*"based" + 0.005*"method" + 0.005*"proceeding" + 0.005*"approach" + 0.005*"network" 2018-01-20 01:00:00,945 : INFO : topic diff=25.860506, rho=1.000000 2018-01-20 01:00:01,534 : INFO : PROGRESS: pass 1, at document #2000/8867 2018-01-20 01:00:10,161 : INFO : optimized alpha [0.015036169206932378, 0.014620147032871365, 0.01316184027461851, 0.015654795203593797, 0.013465080302907698, 0.014096893016290402, 0.013639774928047139, 0.014386778766377433, 0.013396265494725881, 0.015330366862784753, 0.014211342914053279, 0.014541896865487772, 0.013821965871595241, 0.013150262573416302, 0.013373743321679584, 0.015568978441787727, 0.013911498204904199, 0.015274960753643924, 0.014290249223998427, 0.01327315183047394, 0.014054220524267606, 0.014221522051106342, 0.014468910078071256, 0.013927649499424825, 0.013728696835525541, 0.01435979024405582, 0.014989811578068243, 0.014407579830817547, 0.013101269403501575, 0.013356559588453104, 0.01487340849353868, 0.014993317001351723, 0.01330990678787175, 0.015337846173978731, 0.013500210203638431, 0.0133920341754784, 0.01346324161308547, 0.015401468913056465, 0.014158129041884456, 0.014103757034011365, 0.013944629804725529, 0.013784947486802463, 0.014361261674434404, 0.013607458092521908, 0.016539072963927398, 0.013847920575456819, 0.014393062038772423, 0.013361696202235197, 0.013212047832670208, 0.013673643216077995] 2018-01-20 01:00:10,163 : INFO : PROGRESS: pass 1, at document #4000/8867 2018-01-20 01:00:18,805 : INFO : optimized alpha [0.015084156711368592, 0.014420950366857806, 0.012928554051477208, 0.01552624934890383, 0.013223300927790348, 0.013983101676269562, 0.013430127555943716, 0.014203994616282745, 0.013187855082758439, 0.015131935127142514, 0.014071688631184243, 0.014304362735601268, 0.013656052442934181, 0.012969846868467753, 0.01317679661834914, 0.015359809063171876, 0.013667208158158169, 0.015108033531941173, 0.014098878668482044, 0.013040156499565914, 0.013820596002132656, 0.014047863128428116, 0.014279675287830522, 0.013768674215825852, 0.013540687303029415, 0.014157430655364926, 0.014674526846332022, 0.014169873856300679, 0.012867757295635921, 0.013141881842583573, 0.014678939757465025, 0.014846112150344504, 0.013102964149121394, 0.015184116121036319, 0.013293205174507964, 0.013170214355695879, 0.013263643466366794, 0.015360998503201216, 0.013926490925346542, 0.013908358014486118, 0.013836572258728629, 0.013694478564833993, 0.014220134234429731, 0.013420678512584038, 0.016399563031733823, 0.013652016554348686, 0.014348505269304335, 0.013083256221376202, 0.0129603716137918, 0.013467725098012407] 2018-01-20 01:00:18,806 : INFO : PROGRESS: pass 1, at document #6000/8867 2018-01-20 01:00:28,383 : INFO : optimized alpha [0.01519580103708754, 0.014183490064770189, 0.012653638185936698, 0.015414453851941326, 0.013010733897515656, 0.013829466216323839, 0.013231610854290986, 0.013971011477485558, 0.012981851552039481, 0.015000890986678499, 0.013918039162793048, 0.014135024991120411, 0.013514765542355672, 0.01279328727350725, 0.012970786690876951, 0.015255143416118276, 0.013480955488347606, 0.014912192307688211, 0.01392089388990536, 0.012792954144954558, 0.013593328012272988, 0.013907034146892098, 0.01403730537349795, 0.01361732730768472, 0.013353650827022326, 0.013953213710224965, 0.014459383674931103, 0.014018236290957272, 0.012656174977273074, 0.012946301912456564, 0.014512372074497848, 0.014706833150624456, 0.012935403692228016, 0.01500603806203628, 0.01306919551962054, 0.012987188718914281, 0.013016869377871438, 0.01524634895430646, 0.01370343132678138, 0.013677488575041728, 0.013714358469324932, 0.013566078352845802, 0.014133372885013517, 0.013276785686853134, 0.016249614385588626, 0.013443017951639042, 0.014235832077601526, 0.012820742607643893, 0.012703428821384312, 0.013280375774243942] 2018-01-20 01:00:28,384 : INFO : PROGRESS: pass 1, at document #8000/8867 2018-01-20 01:00:37,937 : INFO : optimized alpha [0.015227341856763521, 0.014002525012998979, 0.012437688020241292, 0.015348037087202948, 0.012797974883945202, 0.013720440838824221, 0.012992655642794133, 0.013767759783978598, 0.012772559151539197, 0.014829523196047651, 0.01378892569789638, 0.01394493300357749, 0.013326891426374056, 0.012643594693245419, 0.012786320010486674, 0.015130440716249803, 0.01323961151481768, 0.014781978881925829, 0.013765745176033019, 0.012599962171991302, 0.013355679396767519, 0.01380118586289842, 0.013808187638691178, 0.0134598014051301, 0.013184425912776274, 0.013806288505336614, 0.014264740055603183, 0.013801368388199352, 0.012460516530228151, 0.012747063515081423, 0.014351473656359562, 0.014598689553237273, 0.012731342493747418, 0.014855978533300479, 0.012824094894583792, 0.012783547429262217, 0.012779732901600373, 0.01511814098232515, 0.013491868406942513, 0.013463880652360278, 0.013575691626918445, 0.013429675085309234, 0.014041161571432249, 0.013076888515208513, 0.016230718185392099, 0.013269155981673554, 0.01414972127357811, 0.012600322051953576, 0.012425665547446865, 0.013064249672112295] 2018-01-20 01:00:37,939 : INFO : PROGRESS: pass 1, at document #8867/8867 2018-01-20 01:00:41,829 : INFO : optimized alpha [0.015261911468908665, 0.013774196068727188, 0.012244708009820771, 0.015261547722163687, 0.012547382420595121, 0.013575430890632014, 0.012770367549015928, 0.013582745597441126, 0.01259589653402555, 0.01466974310028657, 0.013668421304940047, 0.01383864587921153, 0.013152731126284832, 0.012500599493225684, 0.012524592482682857, 0.015080569759809601, 0.013038408381277075, 0.014601223706645045, 0.013561262447031189, 0.012406129801284711, 0.01316281366178527, 0.013694017955488715, 0.01365453777247343, 0.0132862661830663, 0.012940576361664987, 0.013612972652416446, 0.014126637865414826, 0.013621879321672602, 0.012242597078920439, 0.012596384061840674, 0.014127260284637316, 0.014454280352013368, 0.012496013722178166, 0.014730547912453549, 0.012609971690819564, 0.012618069933624226, 0.012554777789796571, 0.015040016472004869, 0.013318570962336514, 0.013252509582435422, 0.013456526628097784, 0.013245432690651903, 0.013936819651824518, 0.012863205185821779, 0.016103011969549681, 0.013097598728973441, 0.014089266071462389, 0.012352771388994079, 0.01217494369608813, 0.012862923359014207] 2018-01-20 01:00:42,816 : INFO : topic #48 (0.012): 0.008*"system" + 0.006*"approach" + 0.006*"language" + 0.006*"problem" + 0.006*"configuration" + 0.005*"two" + 0.005*"user" + 0.005*"international" + 0.005*"based" + 0.004*"present" 2018-01-20 01:00:42,817 : INFO : topic #28 (0.012): 0.009*"model" + 0.008*"system" + 0.006*"analysis" + 0.005*"result" + 0.005*"set" + 0.004*"patient" + 0.004*"gene" + 0.004*"signature" + 0.004*"cell" + 0.004*"used" 2018-01-20 01:00:42,818 : INFO : topic #0 (0.015): 0.019*"berlin" + 0.018*"heidelberg" + 0.017*"springer" + 0.011*"model" + 0.010*"berlin_heidelberg" + 0.010*"data" + 0.008*"query" + 0.008*"proceeding" + 0.008*"springer_berlin" + 0.008*"conference" 2018-01-20 01:00:42,819 : INFO : topic #3 (0.015): 0.019*"model" + 0.013*"system" + 0.011*"translation" + 0.009*"based" + 0.009*"language" + 0.008*"method" + 0.007*"proceeding" + 0.007*"machine" + 0.007*"data" + 0.006*"computational" 2018-01-20 01:00:42,820 : INFO : topic #44 (0.016): 0.015*"model" + 0.009*"language" + 0.009*"system" + 0.007*"using" + 0.007*"data" + 0.006*"method" + 0.006*"based" + 0.005*"approach" + 0.005*"proceeding" + 0.005*"feature" 2018-01-20 01:00:42,826 : INFO : topic diff=5.387338, rho=0.394254 2018-01-20 01:00:43,314 : INFO : PROGRESS: pass 2, at document #2000/8867 2018-01-20 01:00:51,660 : INFO : optimized alpha [0.015372200988167804, 0.013628520427412721, 0.012059417411310714, 0.015136903839331438, 0.012446249599080094, 0.013575115154447339, 0.012618539774091339, 0.013458210503099239, 0.012430073475028846, 0.014492233124500615, 0.013610749072467412, 0.013704035724900858, 0.01303663460635485, 0.012383252754415177, 0.012375544748573551, 0.015065924483839605, 0.012894133737017123, 0.014424089696926025, 0.013430605739981538, 0.01225237219768921, 0.012986023139449566, 0.013646631527348971, 0.013502704846847631, 0.013175374309810138, 0.012801869635021768, 0.013457444260081101, 0.014003980261461269, 0.013548052410588984, 0.012061481874942293, 0.012462322828302095, 0.013905112326525228, 0.01434692057545003, 0.01232195629098313, 0.014699509268216181, 0.012410085965796337, 0.012456698218018409, 0.012414807111545785, 0.015058318090277504, 0.013113236724371077, 0.013120092312954365, 0.013457361012324587, 0.013125863803794711, 0.013872734127545586, 0.012724373353346698, 0.015979966563614143, 0.012925071301572654, 0.01412389152470073, 0.012173088949201841, 0.011992602855176207, 0.012761243809582448] 2018-01-20 01:00:51,662 : INFO : PROGRESS: pass 2, at document #4000/8867 2018-01-20 01:00:59,127 : INFO : optimized alpha [0.015521094939308542, 0.013479267076397523, 0.011903994207435946, 0.015035766213389645, 0.012314044783947486, 0.013537242039180549, 0.012473908472724457, 0.01333154400336547, 0.012290987620865779, 0.014338063281066668, 0.013528807040484044, 0.013533802286302373, 0.012956389819174118, 0.012303793956056183, 0.012273299924616985, 0.014920958246464132, 0.012714668214646759, 0.014300442847657184, 0.013306875117940528, 0.012101148025706887, 0.012822740939735577, 0.013583421306925629, 0.013376782498189384, 0.013067857809805024, 0.012677259773972863, 0.013322743097473019, 0.013751679857978911, 0.013386773263887892, 0.011900962265603542, 0.012344232126548078, 0.013753732205983447, 0.01427244246663556, 0.01217785777152239, 0.014591819694696716, 0.012277033329459721, 0.012298142180034718, 0.012295667053506867, 0.015072425411128758, 0.012938528295398875, 0.012974351704557982, 0.013435973822628667, 0.013098291992758729, 0.01379919537135162, 0.012613977688806953, 0.015867234217575639, 0.012778972876179885, 0.014104399001603174, 0.011978227203823497, 0.011821081292288434, 0.012644901059704316] 2018-01-20 01:00:59,129 : INFO : PROGRESS: pass 2, at document #6000/8867 2018-01-20 01:01:06,720 : INFO : optimized alpha [0.015751058978676312, 0.013308520881177224, 0.011712420354657438, 0.014975562283059404, 0.012202390252266456, 0.013538825644625602, 0.012327261783284907, 0.013162337896646809, 0.012147338252954398, 0.014223031625062964, 0.01342227498553995, 0.013387689517877248, 0.012912239884303905, 0.012231027353929716, 0.012153125015478985, 0.014888272655619508, 0.012578128320975117, 0.014155945504005138, 0.01320008484769074, 0.011943720799305843, 0.012675134474237905, 0.013569326680290289, 0.013182501509334249, 0.012975528467969845, 0.012554268568237076, 0.013181881531162488, 0.013563411511282561, 0.013264958517330156, 0.011756890415304713, 0.012231749326289985, 0.01359711180984024, 0.014172538438189162, 0.01204535369712015, 0.014489189519061547, 0.012105889407789583, 0.012175645726197759, 0.012135649241793759, 0.015043657833569854, 0.01276947081995853, 0.012815930938533362, 0.013409761931098078, 0.013040517939819436, 0.013761821851457913, 0.012542978236364077, 0.015772486493298182, 0.012637280680519902, 0.014046878482324962, 0.011766479314484581, 0.011635665045743898, 0.012517538334515394] 2018-01-20 01:01:06,721 : INFO : PROGRESS: pass 2, at document #8000/8867 2018-01-20 01:01:14,886 : INFO : optimized alpha [0.015941962804287176, 0.013139025095497248, 0.011550347950155549, 0.014908456500190051, 0.012070788121426312, 0.01350899779195068, 0.012160666548755571, 0.012996725624550488, 0.011996380437591011, 0.014089063014285762, 0.013347092603816092, 0.013248865705529992, 0.01279199034518559, 0.012189748082893505, 0.01203458550782343, 0.014789618196328954, 0.012402340446124713, 0.014037995550906929, 0.013116017189380645, 0.011816442073990192, 0.0125129216416528, 0.013572759811741339, 0.012998499599656401, 0.01289087425442253, 0.012449286027397069, 0.013074010244353763, 0.013425992062440752, 0.013113069841500999, 0.01163309184443668, 0.012122430171453718, 0.013469295215453141, 0.014145865700616074, 0.011916776390950535, 0.014378846072082329, 0.011924293548199879, 0.012063134912118485, 0.011969210480952733, 0.015014658513084344, 0.012616031497454503, 0.012657922316171317, 0.01338311091294617, 0.012951256463219428, 0.013705991902943245, 0.012411187212705512, 0.015745603337159391, 0.01251904662803135, 0.01403088918519017, 0.011587118554210624, 0.011427947657896353, 0.012363104807007679] 2018-01-20 01:01:14,887 : INFO : PROGRESS: pass 2, at document #8867/8867 2018-01-20 01:01:18,225 : INFO : optimized alpha [0.016124159285309848, 0.012962270702839942, 0.011440486065739011, 0.014822255907131623, 0.011911576149214132, 0.013453934328948871, 0.012019973262132794, 0.012831564255752412, 0.011840677094078059, 0.013986319837345043, 0.013274289997441357, 0.01313244581598315, 0.012684353543615851, 0.012111268361524685, 0.011834935123130679, 0.014786818678728454, 0.012259329057736976, 0.013887049793562256, 0.01298062500186601, 0.011666996460204013, 0.012387900108661651, 0.013583534524345985, 0.012873810547075051, 0.012824893423627231, 0.012275830787504103, 0.012934533573424244, 0.013297664394028888, 0.012989988380560291, 0.011489241430506445, 0.012063369360997099, 0.013314842585692639, 0.014052982227448283, 0.011747771695002695, 0.014323562032814138, 0.011781940905252706, 0.011973231620470056, 0.011840271556531291, 0.0149995395608399, 0.012504352920742468, 0.012488427947972558, 0.013391546482600566, 0.012815873006905001, 0.013632999528440923, 0.01226149315141792, 0.015623397652593345, 0.012420458790356737, 0.014034150206674388, 0.011404689182587413, 0.011262293809945156, 0.012204005507633032] 2018-01-20 01:01:19,034 : INFO : topic #48 (0.011): 0.008*"system" + 0.007*"configuration" + 0.007*"approach" + 0.006*"language" + 0.005*"grid" + 0.005*"problem" + 0.005*"two" + 0.005*"eye" + 0.005*"user" + 0.005*"international" 2018-01-20 01:01:19,035 : INFO : topic #47 (0.011): 0.011*"language" + 0.007*"model" + 0.006*"query" + 0.005*"design" + 0.005*"using" + 0.005*"proceeding" + 0.005*"retrieval" + 0.005*"network" + 0.005*"information" + 0.005*"ontology" 2018-01-20 01:01:19,036 : INFO : topic #37 (0.015): 0.026*"model" + 0.012*"system" + 0.009*"neural" + 0.009*"data" + 0.007*"learning" + 0.007*"using" + 0.006*"network" + 0.006*"based" + 0.005*"show" + 0.004*"approach" 2018-01-20 01:01:19,037 : INFO : topic #44 (0.016): 0.016*"model" + 0.011*"language" + 0.009*"system" + 0.007*"using" + 0.007*"data" + 0.006*"method" + 0.006*"based" + 0.006*"approach" + 0.006*"proceeding" + 0.006*"computational" 2018-01-20 01:01:19,038 : INFO : topic #0 (0.016): 0.026*"berlin" + 0.025*"heidelberg" + 0.024*"springer" + 0.014*"berlin_heidelberg" + 0.011*"springer_berlin" + 0.010*"model" + 0.010*"data" + 0.009*"proceeding" + 0.009*"international" + 0.009*"query" 2018-01-20 01:01:19,041 : INFO : topic diff=1.895403, rho=0.366778 2018-01-20 01:01:19,442 : INFO : PROGRESS: pass 3, at document #2000/8867 2018-01-20 01:01:26,909 : INFO : optimized alpha [0.016278372414568098, 0.012864428638547915, 0.011316054839522591, 0.014719654766612073, 0.011863491808037911, 0.013543635863354712, 0.011917843668198853, 0.012754854110078118, 0.011726088835734153, 0.013862272898759468, 0.0132767280733151, 0.0130178563891666, 0.012613282163708038, 0.012063194394854895, 0.011737244183289388, 0.01481744985419884, 0.012141119483408359, 0.013754213563530779, 0.012909651730571438, 0.011558517508742185, 0.012271854075013721, 0.013598676648893956, 0.012744490493582946, 0.012742510327829332, 0.012207058830137231, 0.012821604645830631, 0.013206960499078105, 0.012976286321517377, 0.011350655189516336, 0.011979211830532986, 0.013140503668124389, 0.013970104082310973, 0.011619354567520803, 0.014303375583844206, 0.011633608184861135, 0.011848564075723529, 0.01174729708395723, 0.015077108784337429, 0.012349403725029797, 0.012395938376015552, 0.013435070675866908, 0.012735993445610896, 0.013603343184025193, 0.012173191810987007, 0.015549209057796373, 0.012289448256874341, 0.014080664086555436, 0.011273682673716945, 0.011127596036369782, 0.012135205820126057] 2018-01-20 01:01:26,910 : INFO : PROGRESS: pass 3, at document #4000/8867 2018-01-20 01:01:37,894 : INFO : optimized alpha [0.01646523411253701, 0.012750706622161717, 0.011209875302160463, 0.014643951719470602, 0.011799188508468962, 0.013575175970438095, 0.011814437515922923, 0.012661453759610111, 0.011633409275713973, 0.013740171788583161, 0.01326245563817912, 0.012892129112909413, 0.012575819883176223, 0.012030662381270115, 0.011688092524994429, 0.014716877899399999, 0.0120047602963306, 0.013653027096307538, 0.012854530973036356, 0.011463084202211295, 0.012154969241906198, 0.013578583189919096, 0.012651229888021131, 0.012687439390017033, 0.012122815613730532, 0.012712220799866464, 0.013001336830891197, 0.012882948100152381, 0.01122489150145285, 0.01193967377543308, 0.013034182326081072, 0.013911942486830763, 0.011515971295444761, 0.014238688791948378, 0.011534171308941301, 0.011750548183993622, 0.011675593062081911, 0.015139005031238053, 0.012226289472332175, 0.012288371147862005, 0.013442683231422649, 0.012740908718277807, 0.013569302658034962, 0.012111116670773263, 0.015494106514257693, 0.012171955409935778, 0.014086662906132884, 0.011130230938514073, 0.011008877858877346, 0.012061349201437232] 2018-01-20 01:01:37,895 : INFO : PROGRESS: pass 3, at document #6000/8867 2018-01-20 01:01:45,392 : INFO : optimized alpha [0.016747527910003564, 0.012628518913735939, 0.011078190927529132, 0.014610127637866167, 0.011736570398323208, 0.013674380118682329, 0.011691195123475821, 0.012551878540336483, 0.011512991479634508, 0.01366439579875923, 0.013176529133728573, 0.012795942569382509, 0.012574662419231207, 0.01202873372074788, 0.011621563171451152, 0.014692733854609542, 0.011889898314781679, 0.013530497708702469, 0.012776198681308821, 0.011356247680853285, 0.012044172521183142, 0.013622814184482209, 0.01249233450696066, 0.012617732984219584, 0.012043216077359371, 0.012610271435683313, 0.012855261055792851, 0.012822803180858319, 0.011118750591310119, 0.011875314427906354, 0.01291715156030291, 0.01385765642135816, 0.01141766209017939, 0.014190693510063667, 0.011406296816116058, 0.011660073510657044, 0.01156832234785924, 0.015175787808496731, 0.012100109882047296, 0.012162038549020055, 0.013452683537697232, 0.012715676283421845, 0.01356291074648017, 0.012061710149449329, 0.015426401356819856, 0.01206639070479687, 0.014074448329071282, 0.010972268214622934, 0.010870586820562049, 0.011978624815366316] 2018-01-20 01:01:45,393 : INFO : PROGRESS: pass 3, at document #8000/8867 2018-01-20 01:01:53,880 : INFO : optimized alpha [0.016964387229504517, 0.012490253307095077, 0.010954976399617294, 0.014563892739011189, 0.011664895512170335, 0.013734980541645941, 0.011579182936443956, 0.012430159191958615, 0.011408123548435588, 0.013558563171894796, 0.013149777829330129, 0.012677379615677498, 0.012512109553553152, 0.0120438197827496, 0.011567245358492519, 0.014642146569086313, 0.011750634186491828, 0.013443292121889886, 0.012722756253506185, 0.011272864069340253, 0.011940685453515591, 0.013697616115674091, 0.012350425410273999, 0.012572911758171211, 0.011978507693979786, 0.012523452715976428, 0.012748756119293707, 0.012738202759850293, 0.01103936257431142, 0.011831651432460155, 0.012831637476024593, 0.013835278288405551, 0.011340361220432213, 0.014116023508220197, 0.011270209698458598, 0.011578053649559297, 0.011447613460725179, 0.015249681649988986, 0.011984358647175815, 0.012041829324402986, 0.013447476542019877, 0.012683767345287937, 0.013563085002405434, 0.011980645437533032, 0.01541424172060509, 0.011983018183444619, 0.014080177527419592, 0.010831122406548988, 0.010705662004016417, 0.01187248077238329] 2018-01-20 01:01:53,882 : INFO : PROGRESS: pass 3, at document #8867/8867 2018-01-20 01:01:56,945 : INFO : optimized alpha [0.01720385907646849, 0.012370391896214212, 0.010876505049465662, 0.01448352728894667, 0.01154661866553078, 0.013809973314223368, 0.011468711216196211, 0.012292376091810959, 0.011283888097624421, 0.013469965633104378, 0.013133122556085506, 0.012593665106990882, 0.012448235813593358, 0.01201146795072267, 0.011413869192086992, 0.014663928524451853, 0.011640069626189839, 0.013330194294425288, 0.012621589485761397, 0.01118497096995244, 0.011873335570904128, 0.013758706384475849, 0.012256474779960089, 0.012563755831006961, 0.01184441174061391, 0.012423415882166378, 0.012673440045994359, 0.012668976942839277, 0.010930568296944398, 0.011844411506300558, 0.012700598194990666, 0.013806372179411773, 0.011208707505774532, 0.014109402166922918, 0.011178611682264429, 0.01150530750834128, 0.011350136084760459, 0.015300524281426617, 0.011901303112553965, 0.011946096053788985, 0.013479121058015523, 0.012609929377076117, 0.013529346391905717, 0.011886417607566419, 0.015332857926956498, 0.011908766286185439, 0.014107564508771975, 0.010684388480537004, 0.01059424975687865, 0.011757152657053896] 2018-01-20 01:01:57,712 : INFO : topic #48 (0.011): 0.010*"configuration" + 0.008*"system" + 0.007*"approach" + 0.006*"grid" + 0.006*"language" + 0.005*"eye" + 0.005*"two" + 0.005*"problem" + 0.005*"computing" + 0.005*"set" 2018-01-20 01:01:57,713 : INFO : topic #47 (0.011): 0.011*"language" + 0.007*"model" + 0.006*"query" + 0.006*"ontology" + 0.005*"proceeding" + 0.005*"design" + 0.005*"network" + 0.005*"arabic" + 0.005*"using" + 0.005*"retrieval" 2018-01-20 01:01:57,714 : INFO : topic #37 (0.015): 0.028*"model" + 0.012*"neural" + 0.011*"system" + 0.010*"data" + 0.008*"learning" + 0.007*"using" + 0.007*"network" + 0.006*"based" + 0.005*"image" + 0.005*"show" 2018-01-20 01:01:57,714 : INFO : topic #44 (0.015): 0.017*"model" + 0.012*"language" + 0.008*"system" + 0.008*"using" + 0.007*"computational" + 0.007*"data" + 0.007*"method" + 0.006*"linguistics" + 0.006*"proceeding" + 0.006*"based" 2018-01-20 01:01:57,715 : INFO : topic #0 (0.017): 0.032*"berlin" + 0.031*"heidelberg" + 0.031*"springer" + 0.017*"berlin_heidelberg" + 0.013*"springer_berlin" + 0.011*"international" + 0.010*"heidelberg_springer" + 0.010*"proceeding" + 0.010*"conference" + 0.009*"data" 2018-01-20 01:01:57,718 : INFO : topic diff=1.586509, rho=0.344347 2018-01-20 01:01:58,119 : INFO : PROGRESS: pass 4, at document #2000/8867 2018-01-20 01:02:05,377 : INFO : optimized alpha [0.01736770896367465, 0.012313279491426962, 0.010786689129015163, 0.014401344041988198, 0.011533187051482482, 0.013939983606743929, 0.011406874049335474, 0.012240506519324833, 0.011201093743361168, 0.013389086174164889, 0.013171722899435622, 0.01251399912743778, 0.012416372454492642, 0.011994621013084451, 0.011361876264335954, 0.01472731807172647, 0.011562035116293645, 0.013224330659482916, 0.012562086975097853, 0.01110567310405125, 0.011792672030214384, 0.013803595176831909, 0.01215509478625236, 0.012522590873625583, 0.011806566262513141, 0.012354764070900665, 0.012609369942634003, 0.012710050158733594, 0.01082223031698545, 0.011808522137756948, 0.012556882712347244, 0.013753602580388194, 0.011107332075398324, 0.014106765434998362, 0.011070612004875103, 0.011407353405872156, 0.011281569077965129, 0.015403160675704682, 0.011786346924924116, 0.011881681159903928, 0.013531882799015692, 0.012563033073928092, 0.0135149524145372, 0.011829130789332891, 0.015291827026280534, 0.011816496409172159, 0.014149230203039615, 0.010592844294479391, 0.010485396635859366, 0.011715384970825252] 2018-01-20 01:02:05,378 : INFO : PROGRESS: pass 4, at document #4000/8867 2018-01-20 01:02:12,322 : INFO : optimized alpha [0.017570783016356086, 0.012240761785806964, 0.01072656271790672, 0.014336056380658501, 0.011502449363321524, 0.014036747021698878, 0.011321942261855027, 0.012168819317925357, 0.011133668250281012, 0.013299799421110295, 0.013167358469043466, 0.012406580102147187, 0.012429545510437555, 0.011979771304016403, 0.011336103251596964, 0.014642367443352188, 0.011462814544561648, 0.013148512785371984, 0.012527468858573159, 0.011048493632906919, 0.011710425137040265, 0.013853909582690123, 0.012103914841783577, 0.012511869333383715, 0.011761787816872879, 0.012292256507140376, 0.01243496235028827, 0.012670436867941601, 0.010724632572598054, 0.011822974723585386, 0.012479595926269555, 0.013723041098258219, 0.011031361435810698, 0.014057929385274937, 0.010996016316650052, 0.011344336859397029, 0.011229817862104349, 0.015490511678717496, 0.011691076146568171, 0.0117930377861171, 0.013557266176552455, 0.012575706524623302, 0.013505190898307125, 0.011797154553904236, 0.015280935958142543, 0.011721504180550065, 0.014172268795143772, 0.01047337477269723, 0.010390151571088703, 0.011652899739897247] 2018-01-20 01:02:12,324 : INFO : PROGRESS: pass 4, at document #6000/8867 2018-01-20 01:02:19,777 : INFO : optimized alpha [0.017873810236176558, 0.012161691260801841, 0.01063155759038595, 0.014307379163772086, 0.01148976885416576, 0.014201773869952548, 0.01122804320624762, 0.012078041960978758, 0.011044261093768644, 0.013249570512832856, 0.013113480271113587, 0.012350637365148787, 0.012452870940726991, 0.012017371941495972, 0.011307186158526427, 0.014643994397865762, 0.011360885563037928, 0.013052979695074499, 0.012488084363291494, 0.010978204044746703, 0.011626919625560587, 0.013932864351638069, 0.011996434925381855, 0.012467832193036522, 0.011722355762587738, 0.012216720199584773, 0.012323951157942152, 0.012681276478601036, 0.010640493032730621, 0.011813086340676281, 0.012394114304760749, 0.013690764093056178, 0.01095031069859027, 0.01401050474795793, 0.010893635718252354, 0.011283447236479163, 0.011145425916040974, 0.015558654891148348, 0.011594419704656076, 0.011687093334107963, 0.013589529129223104, 0.012574780552994044, 0.01351664629518215, 0.0117702613558725, 0.015252046356965724, 0.011644788531985074, 0.014208378751962075, 0.010349150309935129, 0.010282147176633427, 0.011587949074357892] 2018-01-20 01:02:19,781 : INFO : PROGRESS: pass 4, at document #8000/8867 2018-01-20 01:02:26,947 : INFO : optimized alpha [0.018095185725652316, 0.012070769724711577, 0.01053956572240832, 0.014264534380387319, 0.011454310413118439, 0.014316613681652178, 0.011158817045369899, 0.01197996986532978, 0.010973671860214296, 0.013171351947651969, 0.013108158315222742, 0.012266956111680273, 0.012418471734490576, 0.012076595956834517, 0.011298205835935703, 0.014632746715138168, 0.011257639950292225, 0.012988874222306828, 0.01246446520944499, 0.010924658874447365, 0.011559194713470895, 0.014026740447647909, 0.011886681746155262, 0.012440167220429557, 0.011687645520288031, 0.012154158819813429, 0.01225584118857575, 0.012640033076555137, 0.010592417143053642, 0.0118058123642028, 0.012332467756148689, 0.013695247008794681, 0.010897171787722416, 0.013957423885738857, 0.010788577449414416, 0.011231668675834576, 0.0110611434806497, 0.015662827852623271, 0.011502417973324516, 0.011597991480179507, 0.013600092150453876, 0.012557060692319823, 0.013522678444223295, 0.011703999873273868, 0.01525843918591847, 0.01157200954928506, 0.01423468585509993, 0.010237169813234217, 0.010144457605350985, 0.011509100767771924] 2018-01-20 01:02:26,948 : INFO : PROGRESS: pass 4, at document #8867/8867 2018-01-20 01:02:30,080 : INFO : optimized alpha [0.01839981873177414, 0.011978381028353503, 0.010495322032869918, 0.014184251769139662, 0.011356724893937936, 0.014419959894267784, 0.011087081701865386, 0.011862643989013988, 0.01090170013693809, 0.013102489808068678, 0.013107300676828292, 0.012216481861228522, 0.012383295214959362, 0.012062446587573241, 0.011195633289757763, 0.014651505384366287, 0.011174908781765936, 0.012879195177728139, 0.012375134994863384, 0.010854186495991154, 0.011535539509476851, 0.014098621188764354, 0.011818618437739336, 0.012447591409802212, 0.011564407310767025, 0.012106544855230831, 0.012201340377526204, 0.012610287229528056, 0.01050130701335395, 0.011835496056713361, 0.012237162848349237, 0.013680791185666492, 0.01079351693912965, 0.013959179998447526, 0.010730427089229005, 0.011162028444134478, 0.010995580567566667, 0.015785324089184628, 0.011462245943204673, 0.011532286167091609, 0.013651393071668362, 0.012513553881986793, 0.013500506729678344, 0.011639784490020814, 0.015232328132734072, 0.011520290431410379, 0.014249786775807693, 0.010120004546127622, 0.010061754036358703, 0.011418585142965407] 2018-01-20 01:02:30,784 : INFO : topic #48 (0.010): 0.013*"configuration" + 0.008*"system" + 0.007*"approach" + 0.007*"grid" + 0.006*"eye" + 0.006*"instruction" + 0.005*"two" + 0.005*"language" + 0.005*"computing" + 0.005*"set" 2018-01-20 01:02:30,786 : INFO : topic #47 (0.010): 0.011*"language" + 0.007*"model" + 0.007*"ontology" + 0.006*"query" + 0.006*"arabic" + 0.006*"proceeding" + 0.005*"workshop" + 0.005*"network" + 0.005*"design" + 0.005*"using" 2018-01-20 01:02:30,787 : INFO : topic #44 (0.015): 0.018*"model" + 0.014*"language" + 0.008*"computational" + 0.008*"using" + 0.008*"linguistics" + 0.007*"system" + 0.007*"association" + 0.007*"method" + 0.006*"data" + 0.006*"proceeding" 2018-01-20 01:02:30,788 : INFO : topic #37 (0.016): 0.030*"model" + 0.013*"neural" + 0.011*"system" + 0.010*"data" + 0.009*"learning" + 0.008*"network" + 0.007*"using" + 0.006*"based" + 0.005*"image" + 0.005*"show" 2018-01-20 01:02:30,790 : INFO : topic #0 (0.018): 0.036*"springer" + 0.036*"berlin" + 0.035*"heidelberg" + 0.020*"berlin_heidelberg" + 0.015*"springer_berlin" + 0.012*"international" + 0.011*"heidelberg_springer" + 0.011*"proceeding" + 0.011*"conference" + 0.008*"data" 2018-01-20 01:02:30,793 : INFO : topic diff=1.353837, rho=0.325584 2018-01-20 01:02:31,141 : INFO : PROGRESS: pass 5, at document #2000/8867 2018-01-20 01:02:37,830 : INFO : optimized alpha [0.018540436118057998, 0.011955283927230017, 0.010433863856106397, 0.014119837795959829, 0.011372178084084132, 0.014602699996471456, 0.011038279848041116, 0.011818729899282145, 0.010845523801668581, 0.013049148662682827, 0.013189428079510956, 0.012145494115433322, 0.012369197196640995, 0.012069096166038424, 0.011159309858991761, 0.014715144636828241, 0.011116785957397136, 0.012798457863557862, 0.012349191834866621, 0.010796578132423551, 0.011472107601861261, 0.014144400559132084, 0.01173699946475025, 0.012425983109525489, 0.011553929907259433, 0.012063793936404469, 0.012155063221074297, 0.01267157820395488, 0.010424125766014114, 0.011823257647910574, 0.012117893271568238, 0.013658990582189888, 0.01070908637196117, 0.013943033830599543, 0.010654372131733524, 0.011084206972213191, 0.010948562153309969, 0.01590499278216136, 0.011373505363815295, 0.011496603265174722, 0.013699204818018762, 0.012484165697459637, 0.013501083861289585, 0.011594065279473329, 0.01521673711416655, 0.011467524071390667, 0.014334847250802403, 0.010056028754459104, 0.0099750151054073775, 0.01139994016150922] 2018-01-20 01:02:37,831 : INFO : PROGRESS: pass 5, at document #4000/8867 2018-01-20 01:02:44,554 : INFO : optimized alpha [0.018739528394039365, 0.011908004359126574, 0.010400674784742466, 0.014077602004912034, 0.011369317333857775, 0.014727000983420111, 0.010966200035763015, 0.011760293221052773, 0.010799533150815402, 0.012995624688692831, 0.013204913525462934, 0.012050873073271505, 0.012405392385363182, 0.012062054071972642, 0.011147132440980732, 0.014649921089461632, 0.011040721228932061, 0.012755044797135913, 0.012332042835278599, 0.010764022192866523, 0.011421429819100234, 0.014213925193071466, 0.011704731880273791, 0.012434619662780828, 0.011540199366486838, 0.012035698636771975, 0.01201140551354744, 0.012658235419535112, 0.010352519971647513, 0.011875545267904504, 0.012068481396898333, 0.013672660550839589, 0.010649233263952149, 0.013906266319330729, 0.010604694355591313, 0.011043547604341699, 0.010907859380858155, 0.01598643385703187, 0.01131669741429547, 0.0114345392407358, 0.013738232922986874, 0.012512352565615777, 0.013503932781945199, 0.011578880616423936, 0.015226417361389785, 0.01139669111452392, 0.014362844228089625, 0.0099631596732007704, 0.0099063505318679815, 0.011372636478234683] 2018-01-20 01:02:44,555 : INFO : PROGRESS: pass 5, at document #6000/8867 2018-01-20 01:02:51,424 : INFO : optimized alpha [0.019044045985595275, 0.011863484308208706, 0.010333187022954269, 0.014071375160761365, 0.011381101801191689, 0.014936062567462625, 0.010896046131976471, 0.011689835641132353, 0.010727697404872798, 0.012969073981969875, 0.013181308825370597, 0.011999727837006523, 0.012452987565688963, 0.012114866930950291, 0.011131901824893441, 0.014666637611975971, 0.010961557922252405, 0.012678341852323647, 0.012327982112660063, 0.010720442672899205, 0.011359682292757122, 0.014304510559560843, 0.011615013767377311, 0.012411505038344359, 0.011516262842689966, 0.011987037901631062, 0.011924812247412556, 0.012670219137095567, 0.010284421506820846, 0.011909661134153735, 0.012005077313493149, 0.013664066705113626, 0.010592832646486772, 0.013872819458821632, 0.010529026298752429, 0.011002579939235549, 0.010849044769684419, 0.016044335895281207, 0.011255822653240183, 0.011361061639214402, 0.013796871148371899, 0.012519946327496206, 0.013532924193993254, 0.011557308015401705, 0.015226239907373942, 0.011336763685313054, 0.014410657120647553, 0.0098731001252358903, 0.0098192553276697202, 0.011322380605280862] 2018-01-20 01:02:51,425 : INFO : PROGRESS: pass 5, at document #8000/8867 2018-01-20 01:02:58,747 : INFO : optimized alpha [0.019237744222014949, 0.011812864891441456, 0.0102676649424945, 0.014040279759840724, 0.01139454663859411, 0.015108914191423403, 0.010851720982147393, 0.011604623518203494, 0.010679624541976789, 0.012913919249695363, 0.013188859288049141, 0.011921115627109303, 0.012449384736521683, 0.012195989049564614, 0.011149275444457641, 0.014685104788961652, 0.010885890238689341, 0.012632251272842829, 0.012320107726516731, 0.0106783558529564, 0.011310413508905796, 0.014412102440342909, 0.011535596975777459, 0.01238981651566143, 0.01150196936292572, 0.011938352226267285, 0.011879050317378266, 0.012657001146604101, 0.010247657413334747, 0.011934411906947084, 0.011955460318415278, 0.013714069690710182, 0.010559842016260017, 0.013825208562029212, 0.010445236530363675, 0.010964432769659945, 0.010793192496246426, 0.016167208567045458, 0.011198322876717324, 0.011304202728959784, 0.013807940037151906, 0.012521418383625296, 0.013550474492800915, 0.011509962095639811, 0.015258680497286395, 0.011276460670294676, 0.014432657485956039, 0.0097901078934763408, 0.0097092842391338485, 0.011264414578415071] 2018-01-20 01:02:58,748 : INFO : PROGRESS: pass 5, at document #8867/8867 2018-01-20 01:03:01,599 : INFO : optimized alpha [0.019538955291753594, 0.011766101456352197, 0.010242439772249574, 0.013948679347889403, 0.011336399512055666, 0.015243557512924633, 0.01080265591229496, 0.011507708315449935, 0.010625811349586897, 0.012878399369183927, 0.013210265216590368, 0.011872085403125744, 0.012428634816028279, 0.012201840279349709, 0.011084063588194453, 0.014706488918845104, 0.010840182944275106, 0.012533140817565737, 0.012256652787345576, 0.010628700061143314, 0.011313504063937553, 0.01449415994710072, 0.011487492533378219, 0.012434050517197967, 0.011410958188430916, 0.011915374727613568, 0.011853492571540074, 0.012669948218204118, 0.010184510292527637, 0.011995064540774879, 0.011874807252792307, 0.013744746016614238, 0.010481540661752782, 0.013844446047233498, 0.010420075353264457, 0.010912393443846962, 0.010736970798099016, 0.016297770898075496, 0.011162538902946861, 0.011265129739092621, 0.013862132807365873, 0.012503459012859545, 0.0135480922788028, 0.011461627027009218, 0.015272925046699687, 0.011242112592888136, 0.014484231274988994, 0.0096943413688781489, 0.009647998717693779, 0.011180346991952235] 2018-01-20 01:03:02,235 : INFO : topic #48 (0.010): 0.015*"configuration" + 0.008*"system" + 0.007*"grid" + 0.007*"approach" + 0.007*"instruction" + 0.006*"eye" + 0.005*"embedded" + 0.005*"set" + 0.005*"simulation" + 0.005*"two" 2018-01-20 01:03:02,236 : INFO : topic #47 (0.010): 0.011*"language" + 0.008*"ontology" + 0.007*"model" + 0.007*"arabic" + 0.006*"query" + 0.006*"workshop" + 0.006*"proceeding" + 0.005*"network" + 0.005*"ceur" + 0.005*"latency" 2018-01-20 01:03:02,238 : INFO : topic #44 (0.015): 0.019*"model" + 0.015*"language" + 0.010*"computational" + 0.009*"linguistics" + 0.008*"association" + 0.008*"using" + 0.007*"word" + 0.007*"method" + 0.007*"proceeding" + 0.007*"association_computational" 2018-01-20 01:03:02,239 : INFO : topic #37 (0.016): 0.031*"model" + 0.015*"neural" + 0.010*"data" + 0.010*"system" + 0.009*"learning" + 0.009*"network" + 0.007*"using" + 0.006*"based" + 0.005*"image" + 0.005*"show" 2018-01-20 01:03:02,240 : INFO : topic #0 (0.020): 0.041*"springer" + 0.039*"berlin" + 0.038*"heidelberg" + 0.022*"berlin_heidelberg" + 0.016*"springer_berlin" + 0.014*"international" + 0.012*"heidelberg_springer" + 0.012*"proceeding" + 0.011*"conference" + 0.008*"model" 2018-01-20 01:03:02,243 : INFO : topic diff=1.168602, rho=0.309589 2018-01-20 01:03:02,569 : INFO : PROGRESS: pass 6, at document #2000/8867 2018-01-20 01:03:09,154 : INFO : optimized alpha [0.019664267440587243, 0.011773080382106152, 0.010209237182297796, 0.013903424788802654, 0.01137016333297826, 0.015448278719448927, 0.010766898844062067, 0.011481946670854455, 0.010584398855958823, 0.012834538886810191, 0.013315340306404759, 0.01182312066952382, 0.012440370303929809, 0.012207166637022074, 0.011056514791194008, 0.014774535751721295, 0.010798431346463841, 0.012468573062757341, 0.012256472171925742, 0.010591394288551331, 0.011261301612845718, 0.01455171150282555, 0.011424363068328487, 0.012422837781432184, 0.011426012464254171, 0.011894870038532316, 0.011833084948213014, 0.012742654185810519, 0.010127269827515963, 0.012014397830887062, 0.01178170545982367, 0.013741240018047222, 0.010416134846198973, 0.013829071413707117, 0.010368642159105518, 0.010853735885503115, 0.010705482481625101, 0.0164399536699564, 0.011091937329864448, 0.011239838378960055, 0.013916107122038866, 0.012481044530625741, 0.013569618200509424, 0.011425034710579518, 0.015291863083889459, 0.011209527227271435, 0.01456985652179856, 0.0096554558611520493, 0.0095792393374355168, 0.011177061979785612] 2018-01-20 01:03:09,155 : INFO : PROGRESS: pass 6, at document #4000/8867 2018-01-20 01:03:15,498 : INFO : optimized alpha [0.019855937774073063, 0.011744248722661483, 0.010192643796572858, 0.013869927026930047, 0.011391545198413374, 0.015588999931030991, 0.010705937789007821, 0.011431956338204862, 0.01055216376341137, 0.012815821197380454, 0.013347791560889713, 0.011749932052568308, 0.01248866000306454, 0.012202963089500399, 0.011063408996095926, 0.014707847554667444, 0.010744558741620103, 0.012439975500872807, 0.012258584312733597, 0.010570618457550236, 0.011226024187733843, 0.014648582029370336, 0.011411925181530265, 0.012449749967791824, 0.011427266232019135, 0.011884046729686093, 0.011723426000472563, 0.012757401899865564, 0.010065569884775156, 0.012079581213131676, 0.011749009519100627, 0.013777581450676531, 0.010365283876752231, 0.013794214691140796, 0.010327061112587898, 0.010817925324455989, 0.010681201465916381, 0.016551638294421384, 0.011057268872034783, 0.011199961342525041, 0.013955075405235988, 0.012511422242729399, 0.013578716910092919, 0.011404969602301552, 0.015310969408321298, 0.011156280232832633, 0.014609082439246964, 0.0095822855470821481, 0.0095324212918409482, 0.011156961733182142] 2018-01-20 01:03:15,500 : INFO : PROGRESS: pass 6, at document #6000/8867 2018-01-20 01:03:21,372 : INFO : optimized alpha [0.020127017989605717, 0.011726641432180768, 0.010149781559359776, 0.013872440099597618, 0.011405868155136421, 0.015806346741156753, 0.010646536209725383, 0.011387990502768995, 0.010503588921397749, 0.012802232798106571, 0.01334222998352206, 0.0117059435228142, 0.012545987642712305, 0.012267723764486741, 0.011055365973540371, 0.014726649457308425, 0.010679239535729993, 0.0123821742783692, 0.012270931458206777, 0.010545778743289642, 0.011186842800854236, 0.01473455930095975, 0.011345784504054223, 0.012445989297765841, 0.01142378556848847, 0.011862508638786681, 0.011662245540608827, 0.012800051968904932, 0.010005439180256148, 0.012129011072424768, 0.011708321800101158, 0.013775398258786504, 0.010315930405197765, 0.013768775599630079, 0.010267415070956425, 0.010786676202101619, 0.010646032984803449, 0.016624629507669034, 0.011013330633003478, 0.011152346496056146, 0.014015551784931909, 0.012527835728531563, 0.013619113440644379, 0.011391030514723566, 0.015325145226945764, 0.011109522778372754, 0.014665310750179107, 0.0095169526223817108, 0.0094666662072599474, 0.011128747722383051] 2018-01-20 01:03:21,373 : INFO : PROGRESS: pass 6, at document #8000/8867 2018-01-20 01:03:27,750 : INFO : optimized alpha [0.020320008716041709, 0.011694010947780718, 0.010106717106529169, 0.013849237381499714, 0.011425431506604362, 0.016001354810270992, 0.010617506633807521, 0.011327972334263759, 0.010470377564604857, 0.012749629013640953, 0.013381049291059068, 0.011652224581336674, 0.012555899167390531, 0.012348815754966755, 0.011083308325546727, 0.014754161747792491, 0.010618569922454671, 0.012344876636691487, 0.012286939741362572, 0.010517166781845782, 0.011152181280494443, 0.014850889821079646, 0.011283445328110047, 0.012437572172611787, 0.011430917547968135, 0.011838676862144274, 0.011643712879989344, 0.012814030549692294, 0.0099803041597528105, 0.012163734902161367, 0.011672938818178845, 0.013846417588964923, 0.010292823624595764, 0.01374051816555877, 0.010201491476238422, 0.01076605971462322, 0.010600390531178214, 0.016754414442480301, 0.010978914274622251, 0.011111262451143666, 0.014027690489797436, 0.012534633006141145, 0.013636151098934537, 0.011357980833785501, 0.015370530231497889, 0.011067154940862838, 0.014718793729620686, 0.009456075860105816, 0.0093736992150324147, 0.011086189676003609] 2018-01-20 01:03:27,751 : INFO : PROGRESS: pass 6, at document #8867/8867 2018-01-20 01:03:30,519 : INFO : optimized alpha [0.020619850532263981, 0.011676798820557542, 0.01009264938483662, 0.013786590560892412, 0.011393518932562656, 0.016155862211713477, 0.01058327595473809, 0.011254685111511738, 0.01042364681221147, 0.012728406062528641, 0.013425944047290517, 0.011611459871030431, 0.012573803606414214, 0.012390208444665075, 0.011038241087300253, 0.014788556971213798, 0.010584292125604418, 0.012262418486334276, 0.01225739892322948, 0.010476951693997713, 0.011165930404888131, 0.014937621940924597, 0.011247119576019619, 0.01250953558013855, 0.011353947509787752, 0.011827180276313588, 0.011632369174950412, 0.012819121570846172, 0.0099363687705415916, 0.012235688299091392, 0.011607058483158504, 0.01387807871951941, 0.010238770788536803, 0.01377005668589492, 0.010185670785793173, 0.010720705618556606, 0.010554660367291266, 0.01691663036226243, 0.010964742267981775, 0.011082148620399114, 0.014060706967659967, 0.012525978895572628, 0.013646460773637943, 0.011311359659394735, 0.015420020813137768, 0.011042478909014176, 0.014778122806315859, 0.0093794953725837866, 0.009325614389602414, 0.011018086173554332] 2018-01-20 01:03:31,196 : INFO : topic #48 (0.009): 0.018*"configuration" + 0.008*"system" + 0.008*"grid" + 0.008*"instruction" + 0.007*"approach" + 0.006*"simulation" + 0.006*"embedded" + 0.006*"set" + 0.005*"two" + 0.005*"computing" 2018-01-20 01:03:31,197 : INFO : topic #47 (0.009): 0.011*"language" + 0.010*"ontology" + 0.007*"workshop" + 0.007*"arabic" + 0.007*"ceur" + 0.007*"model" + 0.007*"proceeding" + 0.006*"query" + 0.006*"latency" + 0.005*"network" 2018-01-20 01:03:31,199 : INFO : topic #5 (0.016): 0.018*"system" + 0.018*"web" + 0.015*"service" + 0.012*"acm" + 0.010*"user" + 0.008*"new" + 0.008*"application" + 0.007*"design" + 0.007*"paper" + 0.007*"proceeding" 2018-01-20 01:03:31,201 : INFO : topic #37 (0.017): 0.032*"model" + 0.016*"neural" + 0.011*"data" + 0.010*"system" + 0.009*"learning" + 0.009*"network" + 0.007*"using" + 0.006*"based" + 0.005*"image" + 0.005*"bayesian" 2018-01-20 01:03:31,202 : INFO : topic #0 (0.021): 0.044*"springer" + 0.042*"berlin" + 0.041*"heidelberg" + 0.023*"berlin_heidelberg" + 0.017*"springer_berlin" + 0.015*"international" + 0.013*"heidelberg_springer" + 0.012*"proceeding" + 0.012*"conference" + 0.008*"model" 2018-01-20 01:03:31,205 : INFO : topic diff=1.015123, rho=0.295740 2018-01-20 01:03:31,493 : INFO : PROGRESS: pass 7, at document #2000/8867 2018-01-20 01:03:37,815 : INFO : optimized alpha [0.020759358010078739, 0.011702231808962141, 0.010078349273297274, 0.013749367033806924, 0.011448315297690597, 0.016361749961536286, 0.010558067032627082, 0.011236646381238079, 0.010395603959744501, 0.012701054939861596, 0.013528478389412126, 0.011575372425358827, 0.012584429624681917, 0.012417012992927644, 0.011027169413053447, 0.014865523940833427, 0.010554198810427644, 0.012205271220830901, 0.012273510196107371, 0.010453439743618684, 0.01113530807622963, 0.014987069266971203, 0.011208506389959502, 0.012506484994405582, 0.011379351178246859, 0.01182314497182408, 0.011635741497495434, 0.012907623306531724, 0.009891066137492523, 0.012269940945943091, 0.011528186164426683, 0.013890476391890751, 0.010182893076184847, 0.013767106541735432, 0.010145813572415034, 0.010670281398726814, 0.01054492162995762, 0.017052999940899795, 0.010904991609472768, 0.011077703767827012, 0.014123328698455888, 0.012516918525327811, 0.013669278556075162, 0.011272892063224205, 0.015460475215375928, 0.011023371135156255, 0.014853380778245094, 0.0093508172779115505, 0.0092680178793364022, 0.011019366821654069] 2018-01-20 01:03:37,816 : INFO : PROGRESS: pass 7, at document #4000/8867 2018-01-20 01:03:44,083 : INFO : optimized alpha [0.020932500872626044, 0.011703408177450187, 0.010067757175533192, 0.013720946842277818, 0.011472928696332161, 0.016514466784489115, 0.01050274993772087, 0.011207145637603179, 0.010378902630075508, 0.012702879038170668, 0.013561920444175993, 0.011509623332495415, 0.012641560781732192, 0.012427073226285289, 0.011040801767137067, 0.014805859635495407, 0.01050939604582408, 0.0121834991524061, 0.012286165928418885, 0.010436114452155183, 0.011118747817603806, 0.015077939164031022, 0.011205553187575669, 0.012529336156917268, 0.011396324527161631, 0.011821229189324123, 0.011542855885805956, 0.012934261025888835, 0.0098384803527537239, 0.012347190084939834, 0.011513877177844857, 0.013961601824392954, 0.010146295212497049, 0.013727427951662504, 0.01011773626376933, 0.010645830466955115, 0.010536934051189811, 0.017176667447314956, 0.010886703552558682, 0.011057508280066125, 0.014161207279329411, 0.012549097901195164, 0.01368610871586488, 0.011258301256467268, 0.015484884465804642, 0.010981204973791861, 0.014898614473585252, 0.009296856121470241, 0.0092345595959444898, 0.011003159699329919] 2018-01-20 01:03:44,084 : INFO : PROGRESS: pass 7, at document #6000/8867 2018-01-20 01:03:50,318 : INFO : optimized alpha [0.021208079507655891, 0.011699630961470179, 0.010046806025851026, 0.013728622026764564, 0.011498983436670725, 0.016735581336593787, 0.010454983722722992, 0.011165067265214011, 0.010343629068383547, 0.01270356720171369, 0.01357417466437221, 0.011484407651182374, 0.012701874977958076, 0.012493496899249354, 0.011036959161856097, 0.014830754617029688, 0.0104613156130714, 0.012150295551036885, 0.012313903482152399, 0.010436088447735746, 0.011106219310157281, 0.015155848252787487, 0.01115690166634618, 0.012530020641863295, 0.011405438407692632, 0.011807337304583056, 0.011498642277023635, 0.012989815488946971, 0.009798574975611235, 0.012408990670869187, 0.011476727273203161, 0.01398048008144308, 0.010108031240346779, 0.013710851499813364, 0.01007542296476838, 0.010627000193925715, 0.010518420558497683, 0.017254302865433051, 0.010851486590698681, 0.011028055325922478, 0.014222105976850844, 0.012568009138982423, 0.013731186897750824, 0.011254394310236435, 0.015524946785135912, 0.01094029099998374, 0.014943309849465552, 0.0092471349292907491, 0.0091872407182463749, 0.0109904429551643] 2018-01-20 01:03:50,320 : INFO : PROGRESS: pass 7, at document #8000/8867 2018-01-20 01:03:56,789 : INFO : optimized alpha [0.021398719690197746, 0.011693166004663807, 0.010020883770555466, 0.013711971835429902, 0.011525210126588305, 0.016933755572968692, 0.010432663015803459, 0.011111110409332119, 0.010328578862280592, 0.012670575446049744, 0.013629676844841518, 0.011436411913661012, 0.012731060520564242, 0.012585137160197241, 0.011077173633959176, 0.014859895222701688, 0.010418756908401059, 0.012123378516158915, 0.012343256477273152, 0.010427599462131549, 0.011084565170120951, 0.015283855686479079, 0.011108241875470203, 0.012526081213147864, 0.011416313849062094, 0.011796237306779984, 0.011487743717640896, 0.013017190510672684, 0.0097868424359884524, 0.012465160207997747, 0.011454585554165267, 0.014088804276149246, 0.0100868362426988, 0.013679522001281186, 0.010026859298427693, 0.010624212496620197, 0.010488280768082133, 0.017388579077397329, 0.01082199033859947, 0.011002640459882097, 0.014236172967513488, 0.012581445448792345, 0.01375576498632046, 0.011217649382509468, 0.015594687956084832, 0.010903350828906096, 0.015003046046107098, 0.009199912045770765, 0.0091089899417050417, 0.010955874572140223] 2018-01-20 01:03:56,790 : INFO : PROGRESS: pass 7, at document #8867/8867 2018-01-20 01:03:59,596 : INFO : optimized alpha [0.02170594926655774, 0.011712517702254453, 0.010015790101996318, 0.013669839304124307, 0.011512180791037024, 0.017079944043548553, 0.01041054040453023, 0.011046249319382502, 0.010286969724278812, 0.012669339804965242, 0.013692097970863723, 0.011398370527193285, 0.012758559781682992, 0.012617345821248117, 0.011038237908423338, 0.014901194774365144, 0.010396847016698489, 0.01207862996238364, 0.012326663899660785, 0.010394649055837073, 0.011106549478634856, 0.015386111829963783, 0.011088344318885503, 0.012632283939669144, 0.01134328012027795, 0.011790301898450989, 0.01151472969431902, 0.013027689994962793, 0.0097487901229090564, 0.012543127542194762, 0.011404479299468815, 0.014100636015767529, 0.010039533681202922, 0.013709462684910736, 0.010021669780049874, 0.010573299000252329, 0.010461012619410823, 0.01753504820396206, 0.010822410851355553, 0.011002992846254903, 0.014264178734531143, 0.012585224383700874, 0.013763786763323287, 0.011179989838311741, 0.01563296188320084, 0.010875709531014882, 0.015060454160496714, 0.0091385718444809208, 0.0090741645422584179, 0.010894022534522287] 2018-01-20 01:04:00,186 : INFO : topic #48 (0.009): 0.020*"configuration" + 0.008*"instruction" + 0.008*"system" + 0.008*"grid" + 0.007*"approach" + 0.007*"simulation" + 0.006*"embedded" + 0.006*"set" + 0.006*"instruction_set" + 0.006*"space" 2018-01-20 01:04:00,187 : INFO : topic #47 (0.009): 0.012*"ontology" + 0.011*"language" + 0.008*"ceur" + 0.008*"workshop" + 0.007*"arabic" + 0.007*"proceeding" + 0.006*"model" + 0.006*"query" + 0.006*"latency" + 0.005*"packet" 2018-01-20 01:04:00,188 : INFO : topic #5 (0.017): 0.019*"system" + 0.018*"web" + 0.016*"service" + 0.012*"acm" + 0.010*"user" + 0.008*"new" + 0.008*"application" + 0.007*"design" + 0.007*"paper" + 0.007*"proceeding" 2018-01-20 01:04:00,189 : INFO : topic #37 (0.018): 0.033*"model" + 0.017*"neural" + 0.011*"data" + 0.010*"learning" + 0.010*"network" + 0.009*"system" + 0.007*"using" + 0.006*"based" + 0.005*"bayesian" + 0.005*"gaussian" 2018-01-20 01:04:00,191 : INFO : topic #0 (0.022): 0.048*"springer" + 0.044*"berlin" + 0.043*"heidelberg" + 0.025*"berlin_heidelberg" + 0.018*"springer_berlin" + 0.016*"international" + 0.013*"heidelberg_springer" + 0.013*"proceeding" + 0.013*"conference" + 0.008*"model" 2018-01-20 01:04:00,194 : INFO : topic diff=0.884174, rho=0.283598 2018-01-20 01:04:00,474 : INFO : PROGRESS: pass 8, at document #2000/8867 2018-01-20 01:04:06,667 : INFO : optimized alpha [0.02182725180183543, 0.011751659491649651, 0.010014852118760917, 0.013644773034589396, 0.011580034668206573, 0.017289743712256191, 0.010395551646058496, 0.01104534158497966, 0.010269974299591511, 0.01265011967478602, 0.013805506973436759, 0.011371457472248044, 0.012776736778938961, 0.012645260391313489, 0.011044905915265956, 0.014982664821751391, 0.010383179983712533, 0.012040009238615991, 0.012357171923906998, 0.010383951066656601, 0.011077109299744975, 0.015447481607424544, 0.011065575201419087, 0.012636384093437878, 0.011374673279018734, 0.011800780510962183, 0.0115332059771086, 0.013123738930867771, 0.0097185594832239104, 0.012599892663331336, 0.01134648014931309, 0.014128907703905508, 0.0099929418376517708, 0.01371411563623803, 0.0099950644594780976, 0.010533969211528001, 0.010456573133113252, 0.01767415863250979, 0.010780449222588258, 0.011021866916715686, 0.014327648878251492, 0.012577587866299623, 0.013790065558180405, 0.011155237352385593, 0.015675219430741603, 0.010868288344872421, 0.015132897731426991, 0.0091203855978043459, 0.0090277585542111287, 0.01090525125782091] 2018-01-20 01:04:06,668 : INFO : PROGRESS: pass 8, at document #4000/8867 2018-01-20 01:04:12,818 : INFO : optimized alpha [0.021986559990129384, 0.011772232921105939, 0.010017106138591839, 0.013635333829917487, 0.011611969226939247, 0.017425836179871129, 0.010348873623156616, 0.011017913422001991, 0.010262072927795891, 0.012658759188952441, 0.013852810419638047, 0.011313217274014896, 0.012827678551612428, 0.012649575446889313, 0.01106282144299389, 0.014935710254906233, 0.01035242228194224, 0.012029578624165788, 0.012377132340769039, 0.010373191808682364, 0.011082731340056039, 0.015543360072094522, 0.011070075122449393, 0.012670914497139975, 0.011405693284988675, 0.011811326910483153, 0.011454928880647993, 0.013157387378315667, 0.0096784228980091608, 0.012686697958524435, 0.011336411736069868, 0.014198392411410998, 0.0099615692951653869, 0.013691061117086786, 0.0099724184892296397, 0.010511847487756036, 0.010456508923661898, 0.017782786619118335, 0.010773054055676436, 0.011014668784118713, 0.014362999729476651, 0.012609633399544677, 0.013823857552493758, 0.011151313916278884, 0.015712880177115744, 0.010825176201961049, 0.015186761770499609, 0.009078262871084368, 0.0090049194967376945, 0.010890529921100725] 2018-01-20 01:04:12,819 : INFO : PROGRESS: pass 8, at document #6000/8867 2018-01-20 01:04:19,082 : INFO : optimized alpha [0.022231770355618206, 0.011787689860669448, 0.010013538273070414, 0.013648700310858073, 0.011645285504155747, 0.017641593020348456, 0.010309258257148651, 0.010981700189286953, 0.010243476763907477, 0.012676706963834683, 0.013871361111331445, 0.011294684631965054, 0.012901698252530626, 0.012716357790595611, 0.011059353640834491, 0.014964204580083787, 0.010308291460932902, 0.012016455612235256, 0.012411517704363164, 0.010386174387249177, 0.011078034819293023, 0.015624302524839612, 0.011037370258083044, 0.012681962148553225, 0.011427587888760004, 0.011810229882329768, 0.011422718413608641, 0.013217637562087716, 0.0096447195752300697, 0.012754504214166578, 0.011314023669323867, 0.014230749378929142, 0.0099299060045268152, 0.01367844849421395, 0.0099431498613807394, 0.010493559230462135, 0.010453567189681535, 0.017858603660541473, 0.010741782095527825, 0.010992261440015947, 0.014418942943439572, 0.012641380960980319, 0.013886712774435673, 0.011166784106250433, 0.015759014209744442, 0.010798313804509106, 0.015235868775886461, 0.009037386192680746, 0.0089701607153478951, 0.010879184396111472] 2018-01-20 01:04:19,083 : INFO : PROGRESS: pass 8, at document #8000/8867 2018-01-20 01:04:25,325 : INFO : optimized alpha [0.02240690316545239, 0.011798550696821267, 0.0099980124142217946, 0.013642608362499897, 0.011683668461123599, 0.017846575972508331, 0.010296578931620812, 0.010937216785400316, 0.010238219255959809, 0.012662316271499373, 0.013934508277595854, 0.011262373872872074, 0.012934502196498315, 0.012813196094679857, 0.011109077344618317, 0.014984210971199424, 0.010275109348583394, 0.012002204793036398, 0.012454628072988768, 0.010379376356129786, 0.011073730398296757, 0.01575514003738861, 0.011001221942279185, 0.012691796608329326, 0.011446228778924768, 0.011806668473170334, 0.011420480011581994, 0.013253912112396533, 0.0096349461976970273, 0.012815155059144703, 0.011296255978226151, 0.014352949736620709, 0.0099128845366340118, 0.013655424041594644, 0.0099049620347589663, 0.010491964319824061, 0.010430622225967722, 0.017996130595954996, 0.010727903619691663, 0.010981283713252198, 0.014435424850592692, 0.012664162769495954, 0.013917890710912401, 0.01113985956779708, 0.015838586875916796, 0.010772015992564317, 0.015312574320475697, 0.009001211279993886, 0.0089049337896209106, 0.010850002113373605] 2018-01-20 01:04:25,327 : INFO : PROGRESS: pass 8, at document #8867/8867 2018-01-20 01:04:28,035 : INFO : optimized alpha [0.022709104602004981, 0.011842841811987016, 0.01001637679987185, 0.013608735473402583, 0.011672724952544122, 0.017984780688121205, 0.010281480277316155, 0.010875648708771307, 0.010216757711703077, 0.012674237556641208, 0.01401767548461174, 0.011237010223060456, 0.012975052869568222, 0.012840893715935301, 0.011092954316728404, 0.015017713600898463, 0.010257013817303449, 0.011950715948517999, 0.012453351579560453, 0.010355740707027768, 0.011106258515317761, 0.015837084819300146, 0.011009400342871675, 0.012809552956905491, 0.011375833171367845, 0.011797557213291014, 0.011444429684667376, 0.013277759484431112, 0.0096045565355710582, 0.012898866289545187, 0.011262735676045958, 0.014359850611489873, 0.0098710993512130529, 0.013685289364451809, 0.0099212106488002511, 0.010462891409802956, 0.010409199400159093, 0.018157580961653177, 0.010723437113300597, 0.011003755410362782, 0.014478259553182044, 0.012674894952734989, 0.013936693315418337, 0.01110918660216401, 0.015871756865244963, 0.010755138486929772, 0.015363365157383601, 0.0089466142196013808, 0.0088752550233977337, 0.010789842761186547] 2018-01-20 01:04:28,569 : INFO : topic #48 (0.009): 0.023*"configuration" + 0.009*"instruction" + 0.008*"grid" + 0.008*"system" + 0.008*"simulation" + 0.007*"approach" + 0.007*"embedded" + 0.006*"set" + 0.006*"simulator" + 0.006*"space" 2018-01-20 01:04:28,570 : INFO : topic #47 (0.009): 0.014*"ontology" + 0.011*"language" + 0.009*"ceur" + 0.009*"workshop" + 0.007*"proceeding" + 0.007*"arabic" + 0.006*"latency" + 0.006*"model" + 0.006*"query" + 0.006*"packet" 2018-01-20 01:04:28,571 : INFO : topic #5 (0.018): 0.019*"system" + 0.019*"web" + 0.016*"service" + 0.012*"acm" + 0.010*"user" + 0.008*"new" + 0.008*"application" + 0.008*"design" + 0.007*"paper" + 0.007*"research" 2018-01-20 01:04:28,573 : INFO : topic #37 (0.018): 0.034*"model" + 0.017*"neural" + 0.011*"data" + 0.010*"network" + 0.010*"learning" + 0.009*"system" + 0.007*"using" + 0.006*"bayesian" + 0.006*"gaussian" + 0.005*"based" 2018-01-20 01:04:28,574 : INFO : topic #0 (0.023): 0.051*"springer" + 0.046*"berlin" + 0.044*"heidelberg" + 0.026*"berlin_heidelberg" + 0.019*"springer_berlin" + 0.017*"international" + 0.014*"heidelberg_springer" + 0.014*"proceeding" + 0.013*"conference" + 0.007*"model" 2018-01-20 01:04:28,576 : INFO : topic diff=0.770374, rho=0.272838 2018-01-20 01:04:28,842 : INFO : PROGRESS: pass 9, at document #2000/8867 2018-01-20 01:04:35,031 : INFO : optimized alpha [0.02281797280278727, 0.011892363304377443, 0.010019994487706133, 0.013582865927542166, 0.011741950741816609, 0.018189605537394196, 0.010276545225419442, 0.010886736465671332, 0.010206128945937623, 0.012659724648353205, 0.014154838789908419, 0.011220561064410425, 0.012993277237089792, 0.012864953220388754, 0.011102195040082656, 0.015099490201684246, 0.010247926792015761, 0.011918249835611574, 0.012490086856574047, 0.010353497814983828, 0.011080028135248445, 0.015887221607480514, 0.01099825176951375, 0.012823121181253521, 0.011418409883422005, 0.011813001486499367, 0.011477876377938514, 0.013386379543580199, 0.0095911126059689735, 0.012961782703350149, 0.01121941592371869, 0.014406881670073776, 0.0098271500750558294, 0.013694737823156909, 0.0099030364693304188, 0.010428936396894543, 0.010414592857147778, 0.018312693677122498, 0.01068538533823638, 0.011040308310378373, 0.014546801252101688, 0.012675024815013059, 0.013971190114526646, 0.011088585622884133, 0.015919962193558616, 0.010757672622993049, 0.015433410991605894, 0.0089334691239148899, 0.0088367657357866017, 0.010807062100113977] 2018-01-20 01:04:35,032 : INFO : PROGRESS: pass 9, at document #4000/8867 2018-01-20 01:04:40,786 : INFO : optimized alpha [0.022966085962576616, 0.011936443841639272, 0.01002932515164009, 0.013583162718645405, 0.011777638618251454, 0.018314583872806749, 0.010238455152232102, 0.010873982322632671, 0.010202701216524007, 0.012673807408719737, 0.014218807602891004, 0.011183168900760595, 0.013052404540128326, 0.012874526248425224, 0.011122292475712378, 0.015058692887137811, 0.010226589238294104, 0.011912530900096092, 0.012528265295681079, 0.010351150250792723, 0.011090464763804115, 0.015982560177448556, 0.011007102097298691, 0.012866421111865914, 0.011451574509449215, 0.011834803986239756, 0.011404307373617778, 0.013428129033878453, 0.0095602295644446039, 0.013051074799857466, 0.011218487266547977, 0.01447085631900288, 0.0098001776313389986, 0.013673629989007776, 0.009889832328065451, 0.010414067389483004, 0.010426870855416208, 0.018429442830226062, 0.010684661268004455, 0.011045656600870663, 0.014575918593156635, 0.012699950758803724, 0.014013051377366709, 0.011093924117361648, 0.015976745297770947, 0.010736001959643851, 0.015485202533755803, 0.0089000698798940153, 0.0088251101653064237, 0.010803782292745879] 2018-01-20 01:04:40,787 : INFO : PROGRESS: pass 9, at document #6000/8867 2018-01-20 01:04:46,346 : INFO : optimized alpha [0.023195061216615957, 0.011970435293809601, 0.010034251440063674, 0.013596062123739594, 0.011823995923029008, 0.018530744500835302, 0.010206910126310023, 0.010842341112189454, 0.010190388918085335, 0.012705455253816941, 0.014250020492079948, 0.011169639264540374, 0.013134162087404875, 0.012942050297681144, 0.011124630869924252, 0.015093109095889259, 0.010189626660626198, 0.011910779640694523, 0.012575218002617109, 0.01036590507330345, 0.011090935922998278, 0.016061036355128313, 0.010986199529579774, 0.012883193409199299, 0.011477316399702115, 0.011835634780761365, 0.011379628076870537, 0.013503546961909038, 0.0095327696870023144, 0.013117977487927774, 0.011198086770638107, 0.014517977170335986, 0.0097794043518044729, 0.013671903263351855, 0.0098648278644097604, 0.010409779545627244, 0.010432061882795598, 0.018508571234501444, 0.010660941254408389, 0.011035971501618414, 0.014631392345916778, 0.012733302186685917, 0.014079358693637691, 0.011102839714833885, 0.016039394729755733, 0.010712176699742122, 0.015534567477436305, 0.0088712520113561951, 0.0087925622408675519, 0.010796319038587293] 2018-01-20 01:04:46,347 : INFO : PROGRESS: pass 9, at document #8000/8867 2018-01-20 01:04:52,142 : INFO : optimized alpha [0.023353705537130915, 0.011985322562206668, 0.010027555027207686, 0.013593800539244862, 0.011872169868743652, 0.018733035356610452, 0.01019830687787333, 0.010801314507846261, 0.01019075652537787, 0.012691955423907801, 0.014319722531838585, 0.011142923593808125, 0.013175344462475815, 0.013032040180891932, 0.011175472804873396, 0.015117180441580868, 0.010165735326745293, 0.011899671718974915, 0.012630565513325347, 0.010375521489235543, 0.011099276956637568, 0.016180564141086796, 0.010961813742756709, 0.012895395597915554, 0.011495339220249262, 0.011837259962893563, 0.011386537952689121, 0.013543965871056125, 0.0095292540246028415, 0.013184869567215009, 0.011203089080438818, 0.014655569449744934, 0.0097645490827717087, 0.013652439735508601, 0.0098377852177468448, 0.010411437967186722, 0.010417813541363247, 0.018649388408658787, 0.010650979661212553, 0.011046078577192359, 0.014640312634561824, 0.012759804622381864, 0.014118416530628114, 0.011074152111278263, 0.016119454713418985, 0.010693283055036879, 0.015619703365121824, 0.0088419148057371864, 0.0087417322932952981, 0.010777367519140754] 2018-01-20 01:04:52,144 : INFO : PROGRESS: pass 9, at document #8867/8867 2018-01-20 01:04:54,672 : INFO : optimized alpha [0.023660877663345039, 0.012049001481803889, 0.010051676343849146, 0.013550516892638599, 0.011859155482314872, 0.018858320623958619, 0.010189019480569722, 0.010752167371268863, 0.010183840679865283, 0.012707784025965256, 0.014404521365639635, 0.011131569549365367, 0.013232236161794291, 0.013060003231749259, 0.011169887694437428, 0.015171749003836922, 0.010147393834479592, 0.011860311317175423, 0.012642841568198183, 0.010359818883652731, 0.011137858250846882, 0.016258735590491184, 0.010987953367720389, 0.013007968429916363, 0.011420021319915161, 0.011832379746209893, 0.011417851024237258, 0.013583967186313857, 0.0095080620933342738, 0.013266369858258164, 0.011169682706053688, 0.014661835841087725, 0.0097304586440107588, 0.013690655368965257, 0.0098579073653551601, 0.010391865151831266, 0.010401197523513244, 0.018802120429642612, 0.010648582794863415, 0.01108806472408428, 0.014693031880755374, 0.012773328079909718, 0.014151570120461719, 0.01106354503764399, 0.016157326898515713, 0.010675476367706747, 0.01566422166821807, 0.008798105907282831, 0.0087214428444736952, 0.010734758369570207] 2018-01-20 01:04:55,179 : INFO : topic #48 (0.009): 0.025*"configuration" + 0.010*"instruction" + 0.009*"grid" + 0.008*"simulation" + 0.008*"system" + 0.007*"approach" + 0.007*"simulator" + 0.007*"embedded" + 0.007*"set" + 0.007*"space" 2018-01-20 01:04:55,180 : INFO : topic #47 (0.009): 0.017*"ontology" + 0.011*"language" + 0.010*"ceur" + 0.010*"workshop" + 0.008*"proceeding" + 0.008*"arabic" + 0.006*"latency" + 0.006*"model" + 0.006*"packet" + 0.006*"ceur_workshop" 2018-01-20 01:04:55,181 : INFO : topic #37 (0.019): 0.035*"model" + 0.018*"neural" + 0.012*"data" + 0.010*"network" + 0.010*"learning" + 0.009*"system" + 0.007*"using" + 0.006*"bayesian" + 0.006*"gaussian" + 0.006*"parameter" 2018-01-20 01:04:55,183 : INFO : topic #5 (0.019): 0.020*"system" + 0.019*"web" + 0.016*"service" + 0.012*"acm" + 0.010*"user" + 0.008*"application" + 0.008*"new" + 0.008*"design" + 0.007*"paper" + 0.007*"research" 2018-01-20 01:04:55,184 : INFO : topic #0 (0.024): 0.053*"springer" + 0.047*"berlin" + 0.046*"heidelberg" + 0.027*"berlin_heidelberg" + 0.019*"springer_berlin" + 0.018*"international" + 0.014*"proceeding" + 0.014*"heidelberg_springer" + 0.014*"conference" + 0.007*"model" 2018-01-20 01:04:55,187 : INFO : topic diff=0.671035, rho=0.263217 2018-01-20 01:04:55,430 : INFO : PROGRESS: pass 10, at document #2000/8867 2018-01-20 01:05:01,045 : INFO : optimized alpha [0.023746940598531233, 0.012113982486498686, 0.010055289479016882, 0.013536432721478461, 0.011924978608776388, 0.019048850443616365, 0.010191431251804779, 0.010767969143907227, 0.010177244346738428, 0.012698932919003625, 0.014540325502105407, 0.011118079153491688, 0.013244792933273531, 0.01307884836877346, 0.011187085863777745, 0.015259552380924758, 0.010147389093114167, 0.011828259771789465, 0.012676616749268533, 0.010354206979037019, 0.011124132264649257, 0.016309291289476138, 0.010981136709572568, 0.013028772196825193, 0.011470311039574315, 0.011855273582913592, 0.01146456959712226, 0.013685224195017404, 0.0094994467940573268, 0.013328603157999507, 0.011133154015960387, 0.014730597875185016, 0.0096946678723686865, 0.01371138764918187, 0.0098481676156570684, 0.010364859483224504, 0.010409798365474529, 0.018953679049763632, 0.010620718261446454, 0.011131778710498407, 0.014755278625101688, 0.012769249767694484, 0.014197161243907019, 0.011050792487720187, 0.016217070922239001, 0.010685274072906427, 0.01573021549000013, 0.0087946441905498197, 0.0086905257956133922, 0.010754165147173126] 2018-01-20 01:05:01,047 : INFO : PROGRESS: pass 10, at document #4000/8867 2018-01-20 01:05:06,608 : INFO : optimized alpha [0.023876552309699445, 0.012157956986102293, 0.010079991539841621, 0.013548943500225929, 0.011962632371278229, 0.019171175702164332, 0.010158084404613025, 0.010762398128734087, 0.010180230445015273, 0.012716038438409931, 0.014603284277816028, 0.011085932410772874, 0.013306415490792341, 0.013088720803959354, 0.011211917862975502, 0.015227717536722137, 0.010126327637950105, 0.011829598325998463, 0.012718510866512565, 0.010353899211467779, 0.01114305436830185, 0.016394219291674925, 0.010992006769680762, 0.013072188799115802, 0.01151433694375468, 0.01187815758380939, 0.01140051283185858, 0.0137167446867907, 0.0094764971059871344, 0.013427457408769489, 0.011138566542987454, 0.014811861141383517, 0.0096749611300189654, 0.013696970443579651, 0.0098405350479424756, 0.010358758866431672, 0.01043174064999608, 0.01908168034771646, 0.010628461037568747, 0.011153733148310523, 0.014782874955224281, 0.012789384607238351, 0.014248093693987555, 0.011058540489949333, 0.016287468923643713, 0.010678629223072757, 0.015786641430519235, 0.008768351046268847, 0.0086828059887418679, 0.010762219117817556] 2018-01-20 01:05:06,609 : INFO : PROGRESS: pass 10, at document #6000/8867 2018-01-20 01:05:12,172 : INFO : optimized alpha [0.024094294038997471, 0.01219038717036341, 0.010096469870818924, 0.013559474958632857, 0.012018537349545338, 0.019368041547408774, 0.010128256305701527, 0.010738693004040114, 0.010181054289451886, 0.012756944333276095, 0.014647142194972308, 0.011079351043445726, 0.013386863665210582, 0.013148914160218423, 0.011216444845805647, 0.015269271737993853, 0.010095542360591373, 0.011828498169314073, 0.012773825934663743, 0.010368791255248356, 0.011144956320704296, 0.016466899075170863, 0.010981321974512974, 0.013103158830967075, 0.011545099685748524, 0.011886784664632241, 0.011387801754448715, 0.013805048737003955, 0.0094627765071119644, 0.013496944489645644, 0.011117911616700688, 0.014868815224708022, 0.0096573717663914066, 0.01369770793465537, 0.009822777021651647, 0.010349325641172181, 0.010437335621751791, 0.019176929103115078, 0.010609915970190009, 0.011163777492797579, 0.014838284795252595, 0.012829525528282152, 0.014324681981314634, 0.011068381890878422, 0.016346407920384252, 0.01066139904039983, 0.01583665375963008, 0.0087417933740031838, 0.0086573730874338391, 0.010754886553794044] 2018-01-20 01:05:12,173 : INFO : PROGRESS: pass 10, at document #8000/8867 2018-01-20 01:05:17,874 : INFO : optimized alpha [0.02423549790252634, 0.012227765246826731, 0.010100190948929848, 0.013560738529112906, 0.012077821214527872, 0.019569183942009474, 0.010133557743205667, 0.010704418643122039, 0.010187465935898017, 0.012752249384754446, 0.014727358963173063, 0.011055825341624899, 0.01343098785825602, 0.013229586788759466, 0.011269735752337966, 0.015294529455354495, 0.01007848548241325, 0.011820018273093298, 0.012842499221501366, 0.010380062538126629, 0.011163155835718797, 0.016591062184008489, 0.010958631192226865, 0.013119102895230307, 0.011563909110205826, 0.011889949279306505, 0.011409899802558684, 0.013856779351140134, 0.0094620694483127602, 0.013578910190453137, 0.011126198353957222, 0.015019055465274187, 0.0096478839007665407, 0.013677729127347108, 0.0098042226830678952, 0.01035641639710639, 0.010432252773674358, 0.019318944019447632, 0.010612737114935539, 0.011187002602427439, 0.014848084134386676, 0.012857957450652263, 0.014354029927564502, 0.011053545045857477, 0.016433123231407058, 0.010648733593704102, 0.015913722682282222, 0.0087192512433676619, 0.0086166852434666451, 0.010740775762026563] 2018-01-20 01:05:17,875 : INFO : PROGRESS: pass 10, at document #8867/8867 2018-01-20 01:05:20,395 : INFO : optimized alpha [0.024512344697206648, 0.012305976113652166, 0.010126294292622273, 0.013524824091715478, 0.012087871963837414, 0.019682309030167589, 0.010132311862550846, 0.010656274042723684, 0.010178447178654562, 0.012767525271055942, 0.014809463338980162, 0.011056868530492359, 0.013483627282914535, 0.01325853548554524, 0.011260284503280182, 0.01533161270406605, 0.010062757603750583, 0.011795100962060174, 0.012863543930921281, 0.010368607134960501, 0.011213816372754284, 0.016645791514470341, 0.01098135383645843, 0.013234915409368744, 0.011507743284994659, 0.011903029912864202, 0.011444058392561594, 0.013904205230480145, 0.0094459985232580087, 0.013658211196765913, 0.011099365778605791, 0.015051149307758086, 0.0096203583566870209, 0.013719381546555248, 0.0098334375176405264, 0.010339011634494247, 0.010419712748430994, 0.019486827120230848, 0.010621182168057876, 0.011246919351185587, 0.014888080044941847, 0.012870413009934337, 0.01440059362106602, 0.0110344224895237, 0.016465390926210619, 0.010636138723758732, 0.015948410090599805, 0.0086821004897125794, 0.0086172276988059487, 0.010694647844550592] 2018-01-20 01:05:20,866 : INFO : topic #48 (0.009): 0.026*"configuration" + 0.010*"instruction" + 0.009*"grid" + 0.009*"simulation" + 0.008*"simulator" + 0.008*"system" + 0.008*"approach" + 0.007*"embedded" + 0.007*"space" + 0.007*"set" 2018-01-20 01:05:20,867 : INFO : topic #47 (0.009): 0.019*"ontology" + 0.011*"language" + 0.011*"ceur" + 0.011*"workshop" + 0.008*"proceeding" + 0.008*"arabic" + 0.006*"latency" + 0.006*"ceur_workshop" + 0.006*"packet" + 0.006*"model" 2018-01-20 01:05:20,868 : INFO : topic #37 (0.019): 0.036*"model" + 0.019*"neural" + 0.012*"data" + 0.011*"network" + 0.010*"learning" + 0.009*"system" + 0.007*"using" + 0.006*"bayesian" + 0.006*"gaussian" + 0.006*"parameter" 2018-01-20 01:05:20,869 : INFO : topic #5 (0.020): 0.020*"system" + 0.019*"web" + 0.016*"service" + 0.011*"acm" + 0.010*"user" + 0.008*"application" + 0.008*"new" + 0.008*"design" + 0.007*"paper" + 0.007*"research" 2018-01-20 01:05:20,870 : INFO : topic #0 (0.025): 0.055*"springer" + 0.049*"berlin" + 0.047*"heidelberg" + 0.027*"berlin_heidelberg" + 0.020*"springer_berlin" + 0.018*"international" + 0.015*"proceeding" + 0.014*"conference" + 0.014*"heidelberg_springer" + 0.007*"verlag" 2018-01-20 01:05:20,872 : INFO : topic diff=0.584080, rho=0.254547 2018-01-20 01:05:21,103 : INFO : PROGRESS: pass 11, at document #2000/8867 2018-01-20 01:05:26,684 : INFO : optimized alpha [0.024597450550842791, 0.01237328434049079, 0.010134510635969663, 0.013516173105946534, 0.01215471697879798, 0.019877050698984593, 0.010143728663098126, 0.010681552835248473, 0.010172788828304597, 0.012770217182637635, 0.01495912668828007, 0.011045713736263941, 0.013499465565623845, 0.013282846267414434, 0.011279411522486104, 0.015411953447429402, 0.010060767936332222, 0.011770534459591605, 0.012905531003735465, 0.010367613916099543, 0.011204337080526535, 0.016697695458244899, 0.010982466219266117, 0.013247656052638944, 0.011563580974922907, 0.011935657371683213, 0.011494188504213531, 0.013993051478401065, 0.0094402197033944726, 0.013719700784417491, 0.011071618984308463, 0.015128792069399792, 0.0095868110038459294, 0.013745271397635412, 0.0098298216173694202, 0.010316930616013319, 0.010434965565518066, 0.019634750329001902, 0.010602004489592306, 0.01129544735217582, 0.014959932074486312, 0.012868929230217143, 0.014454919542841005, 0.011027399143317271, 0.01653220415494475, 0.010644189004538613, 0.016019245543154224, 0.0086849606402267729, 0.0085935930589762753, 0.010720013144020681] 2018-01-20 01:05:26,685 : INFO : PROGRESS: pass 11, at document #4000/8867 2018-01-20 01:05:32,088 : INFO : optimized alpha [0.024726994863074062, 0.012433876990285468, 0.010170929911176687, 0.013538266945364646, 0.012190769884758436, 0.019994860320394787, 0.010117930104153947, 0.010686229131738237, 0.010182732413670998, 0.01279323100178315, 0.015019007013882812, 0.011016569111546552, 0.013562058015963904, 0.01329502793270588, 0.011304569044538125, 0.015390995844172868, 0.010044855003976828, 0.011778048884187785, 0.012957139900729631, 0.010373017981208241, 0.011221050715096131, 0.016773355182677915, 0.011000645420687094, 0.013299441659869782, 0.01160313864222859, 0.011965126769405162, 0.011446758726635732, 0.01403770352252875, 0.0094171064884170739, 0.013819568294732847, 0.0110868714895866, 0.015220744727233528, 0.0095759061889435903, 0.013728375481284605, 0.0098293814056208129, 0.010311100112566791, 0.010457963010015338, 0.019756477358332806, 0.010622479884197985, 0.011322657544537259, 0.014991995975818913, 0.012899241342483811, 0.01451067111840168, 0.011038727006441222, 0.016609725952584371, 0.010647101498376017, 0.016076718855043137, 0.0086680955949841286, 0.0085888577390216218, 0.010729015583768828] 2018-01-20 01:05:32,089 : INFO : PROGRESS: pass 11, at document #6000/8867 2018-01-20 01:05:37,436 : INFO : optimized alpha [0.024948352534533227, 0.012478238822508283, 0.01018772006170823, 0.013556630383524379, 0.012246412057236186, 0.020183496701155025, 0.010096707032292531, 0.01066919079106928, 0.010182922678970516, 0.012842537266631681, 0.015054206373885659, 0.011012033832030725, 0.013636452076942364, 0.013341646907619194, 0.011311256076901971, 0.01543555518365698, 0.010023278068202229, 0.011778575760406364, 0.013013710506929184, 0.010393105301617314, 0.011240841562852681, 0.016843338456195931, 0.010993711204522987, 0.01332207382675786, 0.011640032151635416, 0.011971549745878872, 0.011440323722322579, 0.014136952602038066, 0.0094063920817552254, 0.013895253738498361, 0.011071202827005284, 0.015294235832565447, 0.0095620693240829717, 0.01373131918233174, 0.0098227539383116715, 0.010305933996738691, 0.010459788598969655, 0.019840108654296112, 0.010610674125434656, 0.011345095950413579, 0.015043733071374013, 0.012932677100537807, 0.014582604439979014, 0.011054772973409323, 0.016677757195067127, 0.010638261745567437, 0.016121684120243385, 0.0086529274122785771, 0.0085692642133616756, 0.010729873912321905] 2018-01-20 01:05:37,438 : INFO : PROGRESS: pass 11, at document #8000/8867 2018-01-20 01:05:43,206 : INFO : optimized alpha [0.025089368291602486, 0.012521016696438429, 0.010199659429197577, 0.013564027796982757, 0.012308547654937147, 0.020373768504353129, 0.010100421069156111, 0.010636523816832088, 0.010189791951705655, 0.012839249103901515, 0.015143066890553586, 0.010989772490364341, 0.013687082839638501, 0.013422088718835743, 0.011368446256884138, 0.015462350853018478, 0.0100109771809818, 0.011775012438428913, 0.01307863675580841, 0.010408296379838923, 0.011265141725329924, 0.016957987913377211, 0.010990234622787809, 0.013341862833060431, 0.011662138821821615, 0.01198623784099611, 0.011463384601487405, 0.014185389088098292, 0.0094139784173075391, 0.013969493210422557, 0.011086235661836769, 0.015449146626327223, 0.0095547201854413854, 0.013717748844546886, 0.0098078220015151239, 0.010319235666417445, 0.01045392938074228, 0.019981235509645281, 0.010620788478711049, 0.011379023238490998, 0.01505456782821865, 0.012958163148179368, 0.014608059829689515, 0.011042718520803569, 0.01678153098572574, 0.010632437885145457, 0.016199574317243608, 0.0086349519636154427, 0.0085381447885846673, 0.01071726460600082] 2018-01-20 01:05:43,208 : INFO : PROGRESS: pass 11, at document #8867/8867 2018-01-20 01:05:45,675 : INFO : optimized alpha [0.025337491244012907, 0.012606027640624027, 0.010224513857608666, 0.013538369568641228, 0.012312213300049377, 0.020475509178137636, 0.010103337134928283, 0.010598072968250135, 0.010187545839016564, 0.012872424239502122, 0.015222491030555415, 0.010995743575935697, 0.013735778620305675, 0.013471382103063406, 0.011371792308823776, 0.0155058440165101, 0.009997272622452081, 0.011752608052506129, 0.01311183588960275, 0.01040650145568706, 0.011320447742489765, 0.017020842497806929, 0.011009467877682333, 0.013457326645440342, 0.011609972528644227, 0.012005522510392618, 0.011496816854415774, 0.014236019199257701, 0.0093969281813429655, 0.014042844877107646, 0.011052034808463779, 0.015496807750505586, 0.0095301262685367136, 0.013742668732701127, 0.0098396117390680288, 0.01030660864901152, 0.010454258827482349, 0.020141745284596755, 0.010645057388542458, 0.01145595684882722, 0.015100146772736255, 0.012977259230900826, 0.014654873377649846, 0.01104125413266945, 0.016817234200258841, 0.010627321373400972, 0.016243591149090689, 0.0086057728873498596, 0.0085400647438447098, 0.01068928427066802] 2018-01-20 01:05:46,117 : INFO : topic #48 (0.009): 0.028*"configuration" + 0.011*"instruction" + 0.010*"simulation" + 0.009*"grid" + 0.009*"simulator" + 0.008*"system" + 0.008*"approach" + 0.008*"space" + 0.007*"embedded" + 0.007*"set" 2018-01-20 01:05:46,118 : INFO : topic #47 (0.009): 0.022*"ontology" + 0.012*"ceur" + 0.011*"workshop" + 0.011*"language" + 0.009*"proceeding" + 0.008*"arabic" + 0.007*"latency" + 0.007*"ceur_workshop" + 0.006*"measurement" + 0.006*"packet" 2018-01-20 01:05:46,119 : INFO : topic #37 (0.020): 0.036*"model" + 0.019*"neural" + 0.012*"data" + 0.011*"network" + 0.011*"learning" + 0.008*"system" + 0.007*"using" + 0.007*"bayesian" + 0.006*"gaussian" + 0.006*"parameter" 2018-01-20 01:05:46,121 : INFO : topic #5 (0.020): 0.021*"system" + 0.019*"web" + 0.016*"service" + 0.010*"acm" + 0.010*"user" + 0.009*"application" + 0.008*"new" + 0.008*"design" + 0.007*"paper" + 0.007*"research" 2018-01-20 01:05:46,122 : INFO : topic #0 (0.025): 0.057*"springer" + 0.050*"berlin" + 0.048*"heidelberg" + 0.028*"berlin_heidelberg" + 0.020*"springer_berlin" + 0.019*"international" + 0.015*"proceeding" + 0.015*"conference" + 0.015*"heidelberg_springer" + 0.008*"verlag" 2018-01-20 01:05:46,125 : INFO : topic diff=0.508135, rho=0.246681 2018-01-20 01:05:46,345 : INFO : PROGRESS: pass 12, at document #2000/8867 2018-01-20 01:05:51,911 : INFO : optimized alpha [0.025413839376005502, 0.012681594326038628, 0.010241914177064796, 0.013532382515473704, 0.0123773769841977, 0.020650010843019925, 0.010116679002651291, 0.010627908397276804, 0.010185083712681609, 0.012874440142164991, 0.015378698213572939, 0.010991937210563596, 0.013756602667847511, 0.013494469765984873, 0.011391194986382551, 0.015586520427086725, 0.010004155062331481, 0.011737504565357803, 0.013153652649835286, 0.010409522447077291, 0.011311844014647464, 0.017062006477172303, 0.01102171961534552, 0.013483592363368879, 0.011659839070453544, 0.012039776250200909, 0.011557061338868163, 0.014325046797512831, 0.0093982237986529431, 0.014102125856089073, 0.011032077965190288, 0.015592365691943966, 0.0095030344275756542, 0.013770395964246658, 0.0098437812530794942, 0.010293877330171255, 0.010483389810518075, 0.020274284878965029, 0.010632034464309902, 0.011511622460523081, 0.015171984215977326, 0.012978260622614242, 0.014708877616000822, 0.01104312973659871, 0.016892579674191995, 0.010642402731280287, 0.016321360329739899, 0.0086140404448494731, 0.008523811442215444, 0.010717307859144699] 2018-01-20 01:05:51,913 : INFO : PROGRESS: pass 12, at document #4000/8867 2018-01-20 01:05:57,315 : INFO : optimized alpha [0.025531948517195657, 0.01274746824498545, 0.010281906156788766, 0.013556435275788127, 0.012418169303466285, 0.020759721882305935, 0.010095188958240155, 0.010632640619984562, 0.010195292713495617, 0.012902697229713522, 0.015445238614173292, 0.010970652142669802, 0.013815266780805023, 0.013502314068053693, 0.011416453666721711, 0.015571868915825663, 0.0099950104334153092, 0.011746148437229324, 0.01321951418216278, 0.01042250483237768, 0.011333035524982087, 0.017131168306683934, 0.01104504973088034, 0.013532005507613342, 0.011700860687576755, 0.012069477347982426, 0.011507932579973363, 0.014382334407961198, 0.0093792528581406673, 0.014204985944875655, 0.011044113832344188, 0.015679032272830878, 0.0094999545269872504, 0.013757977935742783, 0.0098449870220618067, 0.010290471771660932, 0.010508404248870926, 0.02039815917132342, 0.010656232572067263, 0.011556022366502008, 0.015204658549369317, 0.013003930506764322, 0.014769628715966784, 0.011057267365546318, 0.016972274732651303, 0.010656328427477794, 0.016381899382410019, 0.0085980585902025782, 0.0085277055226648524, 0.010729427261209333] 2018-01-20 01:05:57,316 : INFO : PROGRESS: pass 12, at document #6000/8867 2018-01-20 01:06:02,767 : INFO : optimized alpha [0.025742955737714832, 0.012803258710505623, 0.010303860347603735, 0.013581771800041225, 0.012476791720084575, 0.02094645260145625, 0.010075644120761774, 0.010618990944565547, 0.01020219557187438, 0.012951842849262721, 0.015481400998977971, 0.010970327878760434, 0.013889072144020562, 0.013545831148554101, 0.011429235863667519, 0.015617528736279685, 0.0099754363290619659, 0.011746543066806767, 0.013283582893014623, 0.010449760790053575, 0.011355767197398225, 0.017192738590063848, 0.011043834859583598, 0.013558179694870057, 0.011740723922387951, 0.012088407097488135, 0.011505844101868182, 0.014483285103827832, 0.0093688271853636703, 0.014281958314564655, 0.011030365020967782, 0.015745348731237247, 0.009492667633767219, 0.013759890360154339, 0.0098402046270549513, 0.01028892895969701, 0.010514060350776981, 0.020483753525150222, 0.010652822953911593, 0.01158145077587391, 0.015256615020730899, 0.013039163323442713, 0.014835396861285287, 0.011077421125501787, 0.017061450251703275, 0.010658685145755971, 0.016413923808289745, 0.0085868136512433826, 0.008511033231847048, 0.010729656115284966] 2018-01-20 01:06:02,769 : INFO : PROGRESS: pass 12, at document #8000/8867 2018-01-20 01:06:08,050 : INFO : optimized alpha [0.025879854329495414, 0.012855817972373548, 0.0103160605776399, 0.013584774351952628, 0.012546328930194138, 0.021142671505178415, 0.010080226585869812, 0.010592695819308134, 0.010216784192568065, 0.012954327978351016, 0.015565586050806124, 0.010952763990844589, 0.013941272328879815, 0.013622636010873509, 0.01149278954050152, 0.015641797527052217, 0.0099684590330355802, 0.011745952241156368, 0.013352794571913434, 0.010477301789121807, 0.011383076808704821, 0.017302973023744336, 0.011043273924217301, 0.013581566374691946, 0.011763126607380582, 0.012103487035794632, 0.011528321340553758, 0.014538992874679102, 0.0093749540625999196, 0.014364475144046734, 0.011048992412354061, 0.015912739321730505, 0.0094904975896788688, 0.013750803422725852, 0.0098321046364265811, 0.010306477607484989, 0.010514721821467827, 0.020617383820675246, 0.010659033660060454, 0.011619677150647247, 0.015274074940386699, 0.013065188095041383, 0.01485549773465069, 0.01106209841648814, 0.017173263676130732, 0.010658679645554758, 0.016502831970455118, 0.008573869393548407, 0.0084863132842193188, 0.010722140554096963] 2018-01-20 01:06:08,051 : INFO : PROGRESS: pass 12, at document #8867/8867 2018-01-20 01:06:10,011 : INFO : optimized alpha [0.026123473736367052, 0.012954366955013055, 0.010342553413201276, 0.013572200963290737, 0.012568571669224763, 0.021240407845898753, 0.010081213472798018, 0.01055987694324177, 0.010212091844662031, 0.012997195667743851, 0.015656506415791901, 0.010956788868521404, 0.01399820382228063, 0.013669149351185943, 0.011504916830633363, 0.015678313021419192, 0.0099592729569012908, 0.011732235867032795, 0.013393706392933576, 0.010481265612749502, 0.011439755039554843, 0.017369769829842258, 0.011074538196397962, 0.013704513741306412, 0.011721370418677027, 0.012128544423351382, 0.011567400676003869, 0.014576240580354314, 0.0093726342368981117, 0.014436105479603192, 0.011023517573013656, 0.01599822674447704, 0.009470893782757896, 0.0137718759272389, 0.0098661590623750402, 0.010295063995327092, 0.010520905739287302, 0.020765442936988045, 0.01068904997313808, 0.01170657065500846, 0.015311924826344185, 0.013097928595489866, 0.014919521810269197, 0.011064355017166468, 0.017217329184301345, 0.010654099988262474, 0.016537379670323402, 0.0085469381083437598, 0.0084939574963403298, 0.010698698832129001] 2018-01-20 01:06:10,441 : INFO : topic #48 (0.008): 0.029*"configuration" + 0.012*"instruction" + 0.011*"simulation" + 0.010*"simulator" + 0.009*"grid" + 0.008*"system" + 0.008*"space" + 0.008*"approach" + 0.008*"embedded" + 0.007*"set" 2018-01-20 01:06:10,442 : INFO : topic #47 (0.009): 0.025*"ontology" + 0.012*"workshop" + 0.012*"ceur" + 0.011*"language" + 0.009*"proceeding" + 0.008*"arabic" + 0.007*"ceur_workshop" + 0.007*"latency" + 0.006*"w" + 0.006*"measurement" 2018-01-20 01:06:10,443 : INFO : topic #37 (0.021): 0.037*"model" + 0.019*"neural" + 0.012*"data" + 0.011*"network" + 0.011*"learning" + 0.008*"system" + 0.007*"using" + 0.007*"bayesian" + 0.006*"gaussian" + 0.006*"parameter" 2018-01-20 01:06:10,444 : INFO : topic #5 (0.021): 0.021*"system" + 0.020*"web" + 0.016*"service" + 0.010*"user" + 0.010*"acm" + 0.009*"application" + 0.008*"design" + 0.008*"new" + 0.007*"paper" + 0.007*"project" 2018-01-20 01:06:10,445 : INFO : topic #0 (0.026): 0.059*"springer" + 0.051*"berlin" + 0.049*"heidelberg" + 0.028*"berlin_heidelberg" + 0.021*"springer_berlin" + 0.020*"international" + 0.015*"proceeding" + 0.015*"conference" + 0.015*"heidelberg_springer" + 0.008*"verlag" 2018-01-20 01:06:10,447 : INFO : topic diff=0.441832, rho=0.239501 2018-01-20 01:06:10,655 : INFO : PROGRESS: pass 13, at document #2000/8867 2018-01-20 01:06:16,136 : INFO : optimized alpha [0.026186593971809755, 0.013051758524751806, 0.010363801234076112, 0.013557372844251811, 0.012634836346011577, 0.021407638709031448, 0.010096339109128837, 0.010596291770069507, 0.010212482706524447, 0.012999854680550631, 0.015810075888890548, 0.010953069243371147, 0.014022321109721108, 0.013689759122425146, 0.011529817541464515, 0.015772864141643361, 0.0099741243438301336, 0.011715954108552872, 0.013433394335305769, 0.010486430137007731, 0.011429114275183054, 0.017407435044851236, 0.011089106920070913, 0.013727692748192308, 0.011777974466354164, 0.012164238389387792, 0.011631832195215071, 0.014667278008549391, 0.0093791677076964317, 0.014496617731813996, 0.01100922042861464, 0.016090131898921903, 0.0094482335162242061, 0.013799673513162735, 0.0098740561538930363, 0.010290835691103511, 0.010554245059098742, 0.020901116515486606, 0.010679063619541525, 0.01178287857422883, 0.015374987092047801, 0.013099554880599195, 0.014976712851059622, 0.011068010840295146, 0.017304895641395233, 0.010673008448656793, 0.016608334061744324, 0.0085570218089572481, 0.008483202452221986, 0.010727893030918867] 2018-01-20 01:06:16,137 : INFO : PROGRESS: pass 13, at document #4000/8867 2018-01-20 01:06:21,699 : INFO : optimized alpha [0.026304791957575551, 0.013116141423445683, 0.010406011802438173, 0.013579922204708189, 0.01267840380394983, 0.02150725651583682, 0.01008004393497032, 0.010604566120744669, 0.01022874391661645, 0.013026770921794042, 0.01588329798197783, 0.010937673226411565, 0.014093873464482756, 0.013701870557015971, 0.011556324379119638, 0.015758417188083431, 0.009967444570403175, 0.011728385850097016, 0.013495396545701683, 0.010493989750888104, 0.011453275803809308, 0.017474915202544503, 0.011124907664799025, 0.013790875501158126, 0.0118244252666556, 0.012196795720406743, 0.011593257734225241, 0.014725039037187419, 0.0093615462128944348, 0.014607693215814916, 0.011028554116166916, 0.016188638700817118, 0.0094431462117090352, 0.01379005845604203, 0.0098824402426422615, 0.01029070287165003, 0.010572323063994075, 0.021018004735379951, 0.010704160487654088, 0.011833947334983004, 0.015410548853442153, 0.013119208096061635, 0.015024565633387301, 0.011087491594343976, 0.017386114909966287, 0.010690745671138964, 0.016678174879046451, 0.0085457195914381834, 0.0084889027182139167, 0.010744114062497815] 2018-01-20 01:06:21,701 : INFO : PROGRESS: pass 13, at document #6000/8867 2018-01-20 01:06:27,027 : INFO : optimized alpha [0.026506837499851776, 0.013173657981876599, 0.010436556863572964, 0.013603780721293833, 0.012741222463683463, 0.021689966438714903, 0.010063150575960586, 0.010596251940684044, 0.010241748286541884, 0.013085029667868174, 0.015907047013317036, 0.010937304914410606, 0.014172103921649836, 0.013737774823098927, 0.01157461764723674, 0.015820298448754006, 0.0099553469143614098, 0.011734160040677281, 0.013566978469695689, 0.010523036577779804, 0.011485782752498055, 0.017528696737342916, 0.011126109107666758, 0.013825263090379476, 0.011867028928317805, 0.012215913979207871, 0.011599176416279269, 0.01483691948491915, 0.0093522527724458661, 0.014682501218629046, 0.011021442052705756, 0.016264645211811389, 0.0094396058667394946, 0.013789343716818687, 0.0098779254443684354, 0.010293520125276096, 0.010585276043908246, 0.021096679751000196, 0.010699660294318254, 0.011860414391833336, 0.015459441173854746, 0.013159201542605307, 0.015097154934528959, 0.01111017556647039, 0.017468435450970099, 0.010700938166686455, 0.01671173179497424, 0.0085369235942127138, 0.0084796750088565274, 0.010748703770309516] 2018-01-20 01:06:27,029 : INFO : PROGRESS: pass 13, at document #8000/8867 2018-01-20 01:06:32,430 : INFO : optimized alpha [0.026640977065610871, 0.013232537368245551, 0.010452576748807292, 0.013609122510675184, 0.012810352365137924, 0.02187695099133645, 0.010069625650493614, 0.01057311757773371, 0.010263471706421876, 0.013097435919303433, 0.016001867476837892, 0.010924108036315822, 0.014222118688529045, 0.01381320048170106, 0.011637478530412068, 0.015841922235155088, 0.0099565550892759657, 0.011736058455003782, 0.013637099713065131, 0.010552331643565074, 0.011518518733409037, 0.017641460211317135, 0.011125280653152277, 0.013855091914627879, 0.011898003425433271, 0.012229862528191617, 0.011629052159598127, 0.014891189206080759, 0.009364496492684074, 0.014762954906522272, 0.01104181443822233, 0.016456073311765025, 0.0094398744851730294, 0.013786243623774239, 0.0098666817606852166, 0.010308861582232669, 0.010583248403158824, 0.021231045327564639, 0.010707077895747576, 0.011912341187793451, 0.015476406974263602, 0.013185499833093518, 0.015119122871934357, 0.011095651697925334, 0.017571276701896687, 0.010699883223191846, 0.016801781714258241, 0.0085324861321583335, 0.008459533449027298, 0.010740517883277567] 2018-01-20 01:06:32,431 : INFO : PROGRESS: pass 13, at document #8867/8867 2018-01-20 01:06:34,826 : INFO : optimized alpha [0.026895707145591835, 0.01331198247178276, 0.010477610647520428, 0.013597534290466335, 0.012822430301002462, 0.021965521643485079, 0.010071405173826325, 0.010545181149598954, 0.010261802597797784, 0.013138591671357452, 0.016086054694857466, 0.010932143020599592, 0.014275195038363027, 0.013864770641014107, 0.011648289215246714, 0.015893210405130819, 0.0099538854063263953, 0.011730294657612746, 0.013693184343726569, 0.010547322103144888, 0.011573273947730115, 0.017697714133412566, 0.011152495109817065, 0.013966287148579198, 0.011855930785502119, 0.012253564292258093, 0.011660517912999544, 0.014927533944693124, 0.0093655342559085617, 0.014820759743847015, 0.011017954225129909, 0.016551558839004252, 0.0094272409193181424, 0.013811228805185682, 0.0098974606636247874, 0.010295654923427563, 0.010591984461709456, 0.021355251150081678, 0.010748295635961426, 0.012005464348337115, 0.015520010062204827, 0.013216874675680886, 0.015166644782309256, 0.011107198635117261, 0.017642537913250572, 0.010704336417250587, 0.016840995256418235, 0.0085120046848986761, 0.0084699946563751745, 0.010721241158428631] 2018-01-20 01:06:35,251 : INFO : topic #48 (0.008): 0.030*"configuration" + 0.013*"instruction" + 0.011*"simulation" + 0.010*"simulator" + 0.009*"grid" + 0.008*"space" + 0.008*"system" + 0.008*"set" + 0.008*"embedded" + 0.008*"approach" 2018-01-20 01:06:35,252 : INFO : topic #47 (0.009): 0.028*"ontology" + 0.013*"workshop" + 0.012*"ceur" + 0.011*"language" + 0.010*"proceeding" + 0.008*"arabic" + 0.007*"ceur_workshop" + 0.007*"latency" + 0.006*"w" + 0.006*"measurement" 2018-01-20 01:06:35,253 : INFO : topic #37 (0.021): 0.037*"model" + 0.020*"neural" + 0.013*"data" + 0.011*"network" + 0.011*"learning" + 0.008*"system" + 0.007*"using" + 0.007*"bayesian" + 0.007*"gaussian" + 0.007*"parameter" 2018-01-20 01:06:35,254 : INFO : topic #5 (0.022): 0.021*"system" + 0.020*"web" + 0.016*"service" + 0.010*"user" + 0.009*"acm" + 0.009*"application" + 0.008*"design" + 0.008*"new" + 0.007*"paper" + 0.007*"project" 2018-01-20 01:06:35,256 : INFO : topic #0 (0.027): 0.061*"springer" + 0.052*"berlin" + 0.050*"heidelberg" + 0.029*"berlin_heidelberg" + 0.021*"springer_berlin" + 0.020*"international" + 0.016*"proceeding" + 0.016*"conference" + 0.015*"heidelberg_springer" + 0.009*"verlag" 2018-01-20 01:06:35,258 : INFO : topic diff=0.384039, rho=0.232914 2018-01-20 01:06:35,462 : INFO : PROGRESS: pass 14, at document #2000/8867 2018-01-20 01:06:40,703 : INFO : optimized alpha [0.026952484017770419, 0.013408118782595315, 0.010503603747251563, 0.01359011277839862, 0.012882588968384751, 0.022131802070488017, 0.010090410735043872, 0.010586396362668086, 0.01026344704084519, 0.013147798198551454, 0.016233621641178805, 0.010933347233022906, 0.014300741227953901, 0.013876636560838901, 0.011677120969002005, 0.015982332863452051, 0.0099725099321811075, 0.011719486968441328, 0.013735436894305193, 0.010551034703707177, 0.011564519524136048, 0.017724040391871969, 0.011179374771032759, 0.01399751715388175, 0.011920308214175359, 0.012290733726622434, 0.011727403352589266, 0.015020584046754973, 0.0093704333136167793, 0.014879413168879451, 0.011004904790478109, 0.016643113503792696, 0.0094074338752484672, 0.013845316131240437, 0.0099031870590356817, 0.010292265296386773, 0.010629381335005701, 0.021487600881215839, 0.010747237279949298, 0.012086806434110565, 0.015585514250509079, 0.013213082313908029, 0.015232687431853745, 0.011114785295742887, 0.017733617552538751, 0.010729092409351458, 0.01690530071526564, 0.0085245635582248853, 0.0084641441270424719, 0.010755160266218181] 2018-01-20 01:06:40,704 : INFO : PROGRESS: pass 14, at document #4000/8867 2018-01-20 01:06:45,831 : INFO : optimized alpha [0.027068209320304713, 0.013479051291031502, 0.010546656297281637, 0.013615925512524019, 0.012924727089551984, 0.022227123150435929, 0.010076471014413429, 0.010601412603453101, 0.010277128666959602, 0.013182319680410459, 0.016302474063830986, 0.010916907784307239, 0.014361809939756696, 0.013888186966850321, 0.011710141015190533, 0.015973316275364451, 0.0099689566545290934, 0.011736493913844858, 0.013807908728741014, 0.010563258705682655, 0.01159004514811748, 0.017788434772857985, 0.01121676398743587, 0.014060355307695008, 0.011967783132647614, 0.012326149868815228, 0.011691786696503632, 0.01506644942742788, 0.0093561025646007896, 0.014990463440252571, 0.011025819438565987, 0.01674906629087778, 0.0094035554541641343, 0.013841141298462003, 0.0099173296054718679, 0.010291643688417089, 0.010653023003677594, 0.021595860698455212, 0.01077545995224969, 0.012137292396102029, 0.015618267401203938, 0.013230202543331905, 0.015285741248162914, 0.011128870335656741, 0.017818411714218663, 0.010752660333742864, 0.016974443065404507, 0.0085144194414778374, 0.0084713462995410216, 0.010772650764402108] 2018-01-20 01:06:45,832 : INFO : PROGRESS: pass 14, at document #6000/8867 2018-01-20 01:06:50,953 : INFO : optimized alpha [0.027261663303345812, 0.013538387911480787, 0.010576994857381703, 0.013644498983458759, 0.012984444587535691, 0.022416947201428335, 0.010065341428086891, 0.010596571582874652, 0.010291167498991869, 0.013241602059216609, 0.01631999193824113, 0.010923877818316868, 0.014437903815983054, 0.013921743034189256, 0.011731054741555001, 0.016037702243805092, 0.0099635467673152515, 0.011748265506837901, 0.013872276112058129, 0.010591611563393919, 0.011623818559857009, 0.017835181192451477, 0.011226600911730474, 0.014092993410557055, 0.012008711183510356, 0.012352603345511673, 0.011699854260071429, 0.015175334892084471, 0.0093562865229877227, 0.015067037165921296, 0.011028620588894299, 0.016828426598183455, 0.0094076323450824565, 0.013845640514213369, 0.0099154447397393, 0.010299840219779661, 0.010661788514050381, 0.021668359578045571, 0.010772139501033607, 0.012170146655391642, 0.0156679056529518, 0.013267343606349253, 0.015356212599312791, 0.011151287897682273, 0.017902422702765992, 0.010760117895842563, 0.017011628777318565, 0.0085096017872333864, 0.0084669324342086343, 0.010773759126395024] 2018-01-20 01:06:50,955 : INFO : PROGRESS: pass 14, at document #8000/8867 2018-01-20 01:06:56,356 : INFO : optimized alpha [0.027381016680597146, 0.013602189263854042, 0.010599083198067861, 0.013658099899883614, 0.013053522232636608, 0.022603350276147176, 0.01007233264594116, 0.010576005145954932, 0.010313757016736272, 0.013254174446923287, 0.016416514197746847, 0.010910748754318583, 0.014487796332168896, 0.013989541545600911, 0.011795867872010927, 0.016064146123633841, 0.009968729466570795, 0.01175766276861657, 0.013938488244033164, 0.010621516562087057, 0.011660540175086647, 0.017949058434005905, 0.011230435150384853, 0.014126089711188224, 0.012040958924952341, 0.012368214289680287, 0.011731404469594708, 0.015233704074640491, 0.0093730365090418204, 0.015156194581000905, 0.011058057540930135, 0.017024510924771762, 0.0094088275135777999, 0.013844527040967973, 0.0099080675863644288, 0.010321319384400443, 0.010663260271467826, 0.021802347704404798, 0.010779277143293792, 0.012217256287642508, 0.015686139669911496, 0.013297320594227042, 0.015385519031339197, 0.011136051443101105, 0.018019309327060667, 0.01076527974105018, 0.017096909003499756, 0.0085050877545788037, 0.0084479274057688986, 0.01076746865700527] 2018-01-20 01:06:56,358 : INFO : PROGRESS: pass 14, at document #8867/8867 2018-01-20 01:06:58,628 : INFO : optimized alpha [0.027618721349216689, 0.013698996268400753, 0.010625453812092383, 0.013654515526951991, 0.013066464148782028, 0.022689280740207443, 0.01008014189920706, 0.010555140575459478, 0.010312211203184055, 0.013314633592933937, 0.016499131784626885, 0.010919548489046279, 0.014541256700823369, 0.014035101190682428, 0.011811475118012331, 0.016121062278850046, 0.0099692846512494417, 0.011752674180980161, 0.014001922403387591, 0.010610810697531012, 0.011713497280408677, 0.018010257700621795, 0.011253652289394825, 0.014252120954479816, 0.012011269225550531, 0.01240058368565251, 0.011774239314771765, 0.015261397814867166, 0.0093820065912280128, 0.015212601621297646, 0.011041290911761696, 0.017139042178508597, 0.0094000078106267933, 0.013869333863895531, 0.009948803072412183, 0.01031172868719228, 0.010663135524241392, 0.021939472317265764, 0.010828171488800827, 0.012303825292234746, 0.015718618253288071, 0.013323960411201041, 0.015429223887132157, 0.011150653937452985, 0.018098110082075413, 0.010769615212698556, 0.017136447375483681, 0.008488214986582459, 0.0084609759124670628, 0.010754881806570585] 2018-01-20 01:06:59,030 : INFO : topic #47 (0.008): 0.031*"ontology" + 0.014*"workshop" + 0.013*"ceur" + 0.011*"language" + 0.010*"proceeding" + 0.008*"arabic" + 0.007*"ceur_workshop" + 0.007*"w" + 0.007*"latency" + 0.006*"measurement" 2018-01-20 01:06:59,032 : INFO : topic #48 (0.008): 0.030*"configuration" + 0.014*"instruction" + 0.012*"simulation" + 0.011*"simulator" + 0.009*"grid" + 0.009*"space" + 0.008*"system" + 0.008*"set" + 0.008*"instruction_set" + 0.008*"embedded" 2018-01-20 01:06:59,033 : INFO : topic #37 (0.022): 0.037*"model" + 0.020*"neural" + 0.013*"data" + 0.011*"network" + 0.011*"learning" + 0.008*"system" + 0.007*"using" + 0.007*"bayesian" + 0.007*"parameter" + 0.007*"gaussian" 2018-01-20 01:06:59,034 : INFO : topic #5 (0.023): 0.021*"system" + 0.020*"web" + 0.016*"service" + 0.010*"user" + 0.009*"application" + 0.008*"acm" + 0.008*"design" + 0.007*"technology" + 0.007*"paper" + 0.007*"project" 2018-01-20 01:06:59,035 : INFO : topic #0 (0.028): 0.062*"springer" + 0.052*"berlin" + 0.051*"heidelberg" + 0.029*"berlin_heidelberg" + 0.021*"springer_berlin" + 0.021*"international" + 0.016*"proceeding" + 0.016*"conference" + 0.015*"heidelberg_springer" + 0.009*"verlag" 2018-01-20 01:06:59,037 : INFO : topic diff=0.333895, rho=0.226843 2018-01-20 01:06:59,238 : INFO : PROGRESS: pass 15, at document #2000/8867 2018-01-20 01:07:04,522 : INFO : optimized alpha [0.027677032089513057, 0.013803110658441829, 0.01065485144177851, 0.013652011933275292, 0.013128164314688663, 0.022844656808590869, 0.010101579270382355, 0.010598360566321281, 0.010320842234201129, 0.013326696512087038, 0.01666155359947465, 0.010931465328426282, 0.014561974854029389, 0.014051576414958914, 0.01184266804134229, 0.016214064992628133, 0.0099890805444847238, 0.011738760979072452, 0.014041996996366981, 0.010617999506264251, 0.011707832762051893, 0.018036196953285514, 0.011277128856494276, 0.014292688263934766, 0.012080390244376658, 0.012437556857605793, 0.011843466526138029, 0.015355153576177196, 0.0093892929263265888, 0.015266028772223985, 0.011025331524829712, 0.01722609651659111, 0.0093847913999688514, 0.013898302630979296, 0.0099610842140076712, 0.010312358357900009, 0.010699747793048563, 0.022076324186585804, 0.010835475269138747, 0.012380515955214796, 0.015784965845574123, 0.013321138413313657, 0.015493468868978512, 0.011160831131404918, 0.018184370993159893, 0.010793833093660148, 0.017198271482566953, 0.0085011510893150451, 0.0084548988628229561, 0.010788412276604891] 2018-01-20 01:07:04,523 : INFO : PROGRESS: pass 15, at document #4000/8867 2018-01-20 01:07:09,699 : INFO : optimized alpha [0.027782124024968095, 0.013884983280677042, 0.010702229563497962, 0.013680607369348841, 0.01317932298030239, 0.022938934577657527, 0.010090798922021872, 0.010616005578763341, 0.010334258948812521, 0.013359240812391671, 0.016737506014236882, 0.010917453167054197, 0.014621291831649726, 0.014064592488161729, 0.011871377339499027, 0.01620808553223637, 0.0099860796384625885, 0.011758670962148506, 0.014121852824098538, 0.01063351332664287, 0.011738554227799803, 0.018095833204099611, 0.011305785757960001, 0.014356638267239803, 0.012134188724950394, 0.012471398243950872, 0.011811658102319357, 0.015407142443585117, 0.0093788028832231893, 0.015373749794092796, 0.011050034452983557, 0.017333585642710099, 0.0093860489830037939, 0.013897732363117629, 0.0099771518462938458, 0.010315624873474668, 0.010716837126970502, 0.02218385519658941, 0.010862928197275247, 0.01242773733749522, 0.015822448538161529, 0.013338912120077753, 0.015539517446317007, 0.011176056856544112, 0.018272626657726627, 0.01081449448356933, 0.017261216621830909, 0.0084929457358462329, 0.008467198364578302, 0.010807068966156842] 2018-01-20 01:07:09,700 : INFO : PROGRESS: pass 15, at document #6000/8867 2018-01-20 01:07:14,771 : INFO : optimized alpha [0.027962115936543769, 0.013956827906850065, 0.010732254264872721, 0.01370878662853236, 0.013238987608157509, 0.02312088018510804, 0.010082724343161044, 0.010613025080736986, 0.010353826199653426, 0.013425313314696664, 0.016746587249005584, 0.010922568192225495, 0.01469388914028758, 0.014091293522921971, 0.011894711735907169, 0.016274635888529967, 0.0099812503051307944, 0.011774724230412941, 0.014196726798734901, 0.010660048877986368, 0.011769528129664449, 0.018142211503780686, 0.011320207014841264, 0.014393696349793435, 0.012175999166935592, 0.012493570141861194, 0.011821546286858095, 0.015513046400313885, 0.0093792814138387859, 0.015448618739408058, 0.011050774121520425, 0.017414261615741528, 0.0093928853833555193, 0.013905547821837156, 0.0099774442762170078, 0.010325167318622978, 0.010727761614557477, 0.02225311862850202, 0.010861810227305981, 0.012454614082023844, 0.015871166156639603, 0.01338242264123134, 0.015604606666659878, 0.011199495249861968, 0.018357810637800346, 0.010829086531218104, 0.017290393620604975, 0.0084917993100499124, 0.0084652972601364802, 0.010807279565997868] 2018-01-20 01:07:14,772 : INFO : PROGRESS: pass 15, at document #8000/8867 2018-01-20 01:07:20,085 : INFO : optimized alpha [0.028071426333754149, 0.014031525199033886, 0.010749367123511157, 0.013723816131022861, 0.013302003332427311, 0.023291173460984752, 0.010093435691193198, 0.010593475352394823, 0.010382896915603634, 0.013442166312444123, 0.016846393169530537, 0.010911734393090422, 0.014740578616783743, 0.014156693356473347, 0.011961521164555358, 0.016301773072236143, 0.0099856598748344887, 0.011788473514487295, 0.014260488124206357, 0.010691700380622414, 0.011807294591662593, 0.018253233271233568, 0.011327509482255317, 0.014426399155696674, 0.012210575688513463, 0.012510783804516767, 0.011862563881844571, 0.01557041936223149, 0.0094001412138461186, 0.01553971615955827, 0.011083794844109797, 0.017608961259426164, 0.0093979174310445859, 0.013909418141863467, 0.009974593984911323, 0.010345350861412177, 0.01072792205882018, 0.022396596124455079, 0.010873365551921066, 0.012498569167248366, 0.015883574541272479, 0.013411341689460687, 0.015636130945929406, 0.011189773096554131, 0.018469676203745886, 0.010834062636808641, 0.017383143443882733, 0.008488074995858224, 0.008446293517181392, 0.01080144412313333] 2018-01-20 01:07:20,086 : INFO : PROGRESS: pass 15, at document #8867/8867 2018-01-20 01:07:22,343 : INFO : optimized alpha [0.028300861659984699, 0.01413813251568938, 0.010782657832929383, 0.013727584064819914, 0.013312268361013901, 0.023382107610229498, 0.010109382961965105, 0.010576398740234107, 0.01038120735824451, 0.01348977238458869, 0.016918328063492617, 0.010923927887287374, 0.014798310699694915, 0.014207608773933207, 0.011981849305227622, 0.016351136907572447, 0.0099866021116504482, 0.011796457880058507, 0.014320069957300076, 0.010681103161610399, 0.011858520079986343, 0.018314626464012395, 0.011358695468062922, 0.014563504511495313, 0.012195747187939521, 0.012544963069602405, 0.011897841883145237, 0.015614201722851462, 0.0094092200970836547, 0.015594856932910326, 0.011073634206787992, 0.017715271667550072, 0.009390057495664272, 0.013933971913236125, 0.010027357133875337, 0.01033101653127461, 0.010730523342616118, 0.022523067666877086, 0.010921049952479792, 0.012588425937474355, 0.015926119930552065, 0.013443890795286469, 0.015684742439210315, 0.011207083174438965, 0.018541182767945044, 0.010843858109506411, 0.01741880430567912, 0.008472309009866584, 0.0084616817121130391, 0.010792340130445124] 2018-01-20 01:07:22,744 : INFO : topic #47 (0.008): 0.034*"ontology" + 0.014*"workshop" + 0.013*"ceur" + 0.011*"language" + 0.010*"proceeding" + 0.008*"arabic" + 0.007*"ceur_workshop" + 0.007*"w" + 0.007*"latency" + 0.006*"measurement" 2018-01-20 01:07:22,746 : INFO : topic #48 (0.008): 0.031*"configuration" + 0.015*"instruction" + 0.013*"simulation" + 0.012*"simulator" + 0.009*"space" + 0.009*"grid" + 0.008*"set" + 0.008*"system" + 0.008*"instruction_set" + 0.008*"embedded" 2018-01-20 01:07:22,748 : INFO : topic #37 (0.023): 0.038*"model" + 0.020*"neural" + 0.013*"data" + 0.012*"network" + 0.011*"learning" + 0.008*"system" + 0.007*"using" + 0.007*"bayesian" + 0.007*"parameter" + 0.007*"gaussian" 2018-01-20 01:07:22,749 : INFO : topic #5 (0.023): 0.022*"system" + 0.020*"web" + 0.016*"service" + 0.010*"user" + 0.009*"application" + 0.008*"design" + 0.008*"technology" + 0.008*"acm" + 0.007*"project" + 0.007*"paper" 2018-01-20 01:07:22,751 : INFO : topic #0 (0.028): 0.063*"springer" + 0.053*"berlin" + 0.051*"heidelberg" + 0.030*"berlin_heidelberg" + 0.022*"springer_berlin" + 0.021*"international" + 0.016*"proceeding" + 0.016*"conference" + 0.015*"heidelberg_springer" + 0.009*"verlag" 2018-01-20 01:07:22,754 : INFO : topic diff=0.290742, rho=0.221222 2018-01-20 01:07:22,952 : INFO : PROGRESS: pass 16, at document #2000/8867 2018-01-20 01:07:28,133 : INFO : optimized alpha [0.028353100184106039, 0.014245292907533725, 0.010814148673665799, 0.013720704254374693, 0.013372580320427607, 0.023545709884143461, 0.010130711546521679, 0.010621452863996173, 0.010396364722896319, 0.013500189579467527, 0.01707142448302032, 0.01093627236035382, 0.014814444075899481, 0.014219278561640573, 0.01201000205056527, 0.01645167673002559, 0.010012943098092857, 0.01178832676325688, 0.01436899585514357, 0.01069163428702327, 0.01185578665096851, 0.018338270646921666, 0.011383841705526252, 0.014606778212374343, 0.012266737506088823, 0.012585903951646467, 0.01196679882794266, 0.01570677355263168, 0.0094196970577609691, 0.015653542881849232, 0.011060788887706483, 0.017801994732156642, 0.0093748651427430826, 0.013967242435162813, 0.010045755776299754, 0.010332082903402293, 0.010767604933339992, 0.022654307027791157, 0.010930258628659316, 0.012663483122927808, 0.015996823103858998, 0.01344197838281252, 0.015750856422397028, 0.011220862278593339, 0.018624930693357231, 0.010871100192631392, 0.017486250698073389, 0.0084874055868507034, 0.0084580459082173546, 0.010826590306517933] 2018-01-20 01:07:28,134 : INFO : PROGRESS: pass 16, at document #4000/8867 2018-01-20 01:07:33,262 : INFO : optimized alpha [0.028436489853020105, 0.014328483285836555, 0.010859893818380522, 0.013748890406747353, 0.013421055535471179, 0.02362104372577789, 0.010126024094108716, 0.010642671948011518, 0.010412730688194032, 0.013533975661495634, 0.01714912301249193, 0.010922023473340954, 0.014868902047963161, 0.014230726995556112, 0.012042319258957952, 0.016448673381171161, 0.01000707587582297, 0.011806739784296486, 0.014448045584963752, 0.010714886841415741, 0.011886406126822988, 0.01839541492397016, 0.011412761627921302, 0.014673324550566091, 0.012321131582208204, 0.012625173417425615, 0.011935965805051827, 0.015762894360538474, 0.0094147894377729618, 0.015766605487954027, 0.011088949637382928, 0.017911256523567693, 0.0093767556310591811, 0.013968375343787856, 0.010061230412973275, 0.010334556272205302, 0.010781920646070441, 0.022756163519002268, 0.010961701938917255, 0.012706271641926788, 0.016033588422377718, 0.013461734355016009, 0.015798899594591666, 0.011239534902244181, 0.018712748984254741, 0.010898348871102767, 0.017541202222854203, 0.0084864951044353008, 0.0084695599016097467, 0.010842736818288864] 2018-01-20 01:07:33,263 : INFO : PROGRESS: pass 16, at document #6000/8867 2018-01-20 01:07:38,508 : INFO : optimized alpha [0.028614608176839206, 0.014404666672427274, 0.010889659067295766, 0.013781210128721107, 0.013482403132608555, 0.023787586574732619, 0.010120539514738385, 0.010643677253678886, 0.010429434158205214, 0.013590425486282228, 0.017159882991310868, 0.010928871545473319, 0.014938300498098623, 0.014261922778618388, 0.012068086931014188, 0.016510218813455158, 0.010007111314676447, 0.011822807515279456, 0.014523764476269081, 0.010745432831535808, 0.011919872645357977, 0.018437510624277408, 0.011430172485857249, 0.014706585870192895, 0.012364998935613913, 0.012654271899974767, 0.011946280543441108, 0.015871153478511781, 0.0094205666765680342, 0.015846641370778868, 0.011094789349616449, 0.017995373442372335, 0.009385109572309893, 0.013974608099703963, 0.010065577562330109, 0.010343213300959305, 0.010794916816296471, 0.022824964050064525, 0.010966204135934439, 0.012735772040735945, 0.0160779322536308, 0.013511369024780336, 0.015855188028211509, 0.011262511474547479, 0.018795549441938068, 0.010909990634942959, 0.017570756712772084, 0.0084877451687520884, 0.0084727494753690583, 0.010844425687129832] 2018-01-20 01:07:38,509 : INFO : PROGRESS: pass 16, at document #8000/8867 2018-01-20 01:07:43,784 : INFO : optimized alpha [0.028721953072187105, 0.014474214533614848, 0.010906589919346763, 0.013800520231452767, 0.01355442641319953, 0.0239480039152025, 0.010132387059123111, 0.0106272313621619, 0.010457843791215893, 0.013616341747313479, 0.017267121998204144, 0.010918900700173612, 0.01498533723825088, 0.01432502726821717, 0.012136822141531001, 0.016536428666011387, 0.010012870321567878, 0.011837813681632859, 0.014591580395390706, 0.010776355143708697, 0.011958779123161149, 0.018551880830005286, 0.011444346573678781, 0.014745290383557069, 0.012397592308004909, 0.012676940150833213, 0.011996517562510051, 0.01591858160921478, 0.0094432206297513227, 0.015934319120261751, 0.011127434142755476, 0.018188964068647426, 0.0093906589496841678, 0.01398162892872384, 0.010067851012522892, 0.010365688639083857, 0.010796154881186655, 0.022955639984868846, 0.010980734164821437, 0.012782275351887532, 0.016089898817793886, 0.013536074105777018, 0.015883701432975263, 0.011251672800058588, 0.018894218534170691, 0.010917013099043815, 0.017663069197334253, 0.0084864662705784917, 0.0084564324340271422, 0.010837824784057756] 2018-01-20 01:07:43,785 : INFO : PROGRESS: pass 16, at document #8867/8867 2018-01-20 01:07:46,042 : INFO : optimized alpha [0.028929480685650197, 0.014586989697919363, 0.010935584913745331, 0.013800698583513972, 0.013565414417338873, 0.024019883109050042, 0.010150868926354664, 0.010610821899463314, 0.010466466476277867, 0.013672387197212423, 0.017350636450458527, 0.010928742612869895, 0.015040058174881025, 0.014370258448684288, 0.01215551315152808, 0.016578557679261766, 0.010016651166219854, 0.011848948345625051, 0.014654507785436916, 0.010765595998658671, 0.012020452509163631, 0.018613689922192499, 0.011483102582533669, 0.014889509799975538, 0.012381288582307479, 0.012712551206426281, 0.012039573592466312, 0.015970084802451098, 0.00945463714933561, 0.015976208768563236, 0.011120505389555464, 0.018296339359919457, 0.0093883322593248126, 0.014005748189308859, 0.010124507579589168, 0.010370433841031954, 0.010804138276373635, 0.023078023779933784, 0.011030045851889419, 0.012862542746791997, 0.016121650493980958, 0.013574275947181358, 0.015933112967054609, 0.011271524317765694, 0.018969130368475563, 0.010937508753411427, 0.017699545850850089, 0.0084736910444178203, 0.0084781600814061702, 0.010831949708882868] 2018-01-20 01:07:46,428 : INFO : topic #47 (0.008): 0.037*"ontology" + 0.015*"workshop" + 0.013*"ceur" + 0.011*"proceeding" + 0.011*"language" + 0.008*"arabic" + 0.007*"ceur_workshop" + 0.007*"w" + 0.007*"latency" + 0.006*"measurement" 2018-01-20 01:07:46,429 : INFO : topic #48 (0.008): 0.031*"configuration" + 0.016*"instruction" + 0.014*"simulation" + 0.012*"simulator" + 0.010*"space" + 0.009*"grid" + 0.009*"set" + 0.008*"system" + 0.008*"instruction_set" + 0.008*"embedded" 2018-01-20 01:07:46,430 : INFO : topic #37 (0.023): 0.038*"model" + 0.020*"neural" + 0.013*"data" + 0.012*"network" + 0.012*"learning" + 0.008*"system" + 0.007*"bayesian" + 0.007*"using" + 0.007*"parameter" + 0.007*"gaussian" 2018-01-20 01:07:46,431 : INFO : topic #5 (0.024): 0.022*"system" + 0.020*"web" + 0.016*"service" + 0.010*"user" + 0.009*"application" + 0.008*"design" + 0.008*"technology" + 0.008*"project" + 0.008*"paper" + 0.007*"research" 2018-01-20 01:07:46,433 : INFO : topic #0 (0.029): 0.064*"springer" + 0.054*"berlin" + 0.052*"heidelberg" + 0.030*"berlin_heidelberg" + 0.022*"springer_berlin" + 0.022*"international" + 0.017*"proceeding" + 0.017*"conference" + 0.015*"heidelberg_springer" + 0.009*"verlag" 2018-01-20 01:07:46,435 : INFO : topic diff=0.254320, rho=0.216000 2018-01-20 01:07:46,624 : INFO : PROGRESS: pass 17, at document #2000/8867 2018-01-20 01:07:51,804 : INFO : optimized alpha [0.028970860410015849, 0.014689420047590824, 0.010968054082634137, 0.013795462084694489, 0.013619973548009399, 0.024169196115703194, 0.010175261779742227, 0.010656448714278264, 0.010479965280518973, 0.013681077404179086, 0.017501830694015674, 0.010941430138816466, 0.015058354224550444, 0.014386587917087331, 0.012188505805880063, 0.016668609762531674, 0.01004598831100223, 0.011841156871690683, 0.014701060597956754, 0.010778014081637708, 0.01202043409310594, 0.018639285415167725, 0.011509614902137125, 0.014935310089641144, 0.012452850391934613, 0.012755890848805124, 0.012111961672373394, 0.016058097439294573, 0.0094660164718008231, 0.016036858942026732, 0.011112824543609857, 0.018384503520744562, 0.0093770210926873016, 0.014038395462747289, 0.010142230438234591, 0.010370566219515657, 0.010842754846232448, 0.023212292508807474, 0.011040959030976547, 0.012940113084757864, 0.016189710188533733, 0.013567163168864084, 0.01599093292053157, 0.01128625489989893, 0.019038240422367552, 0.010970942951558137, 0.017769016136487341, 0.0084942831404483422, 0.0084776089097491833, 0.010870402328274884] 2018-01-20 01:07:51,806 : INFO : PROGRESS: pass 17, at document #4000/8867 2018-01-20 01:07:56,906 : INFO : optimized alpha [0.029046971176150755, 0.014781820952638034, 0.011011059070354731, 0.013827593748428403, 0.013668703351721734, 0.024240272214704948, 0.010172967853349426, 0.010679823874955443, 0.010501418530651351, 0.013712967397702183, 0.017584934777493462, 0.01093284597871667, 0.01511803293826073, 0.014396450249460057, 0.012225617130740517, 0.016667064442554721, 0.010041545521193462, 0.01186192873693764, 0.014773052205029244, 0.01080175655147002, 0.01205339350170823, 0.018694140694349579, 0.011541172781493596, 0.015001090683835743, 0.012511957462435351, 0.012794751192181949, 0.012084338670561249, 0.016109773614398654, 0.0094633094528306372, 0.016146803166471296, 0.011139387230342242, 0.018487624420065442, 0.009379395155474585, 0.01403963369916284, 0.010155938732678057, 0.010377638390942419, 0.010860128720280722, 0.023304062583356915, 0.011073771380988223, 0.01298691493911636, 0.016222546467221977, 0.013591739799645702, 0.016041081315707122, 0.011305747064626896, 0.019120025787344584, 0.010997369972178704, 0.017822128362852966, 0.0084937794075650407, 0.0084873364320477637, 0.010882964236296465] 2018-01-20 01:07:56,907 : INFO : PROGRESS: pass 17, at document #6000/8867 2018-01-20 01:08:02,079 : INFO : optimized alpha [0.029227291090273326, 0.014859097412486371, 0.011045366582365147, 0.013860739254652156, 0.013730135759341776, 0.024395357892408178, 0.010172067700745538, 0.010684530932035193, 0.010520869670295875, 0.013771777634908762, 0.01760261671483403, 0.010940119234195566, 0.01517945320943262, 0.014422864270827557, 0.012250972986430035, 0.016729443733045281, 0.010043973742569061, 0.011884185232293558, 0.014852890525618593, 0.010832730574290359, 0.012087877017018883, 0.018736004116019566, 0.011561271670448405, 0.015038537379615784, 0.012556545419448159, 0.01282353862935424, 0.012093425371086392, 0.016217080171956701, 0.0094680244112589317, 0.016221945868445133, 0.011145276860518322, 0.018565394174883697, 0.0093880877421921253, 0.01404584598911272, 0.010164108311412203, 0.010386289141370788, 0.010872675883337653, 0.023352593552696279, 0.011076454970905777, 0.013016232266072743, 0.016264709721460227, 0.013642809705843579, 0.016091028820457997, 0.011330724488591784, 0.019211342909780615, 0.01101900675061112, 0.017844283469955022, 0.0084953526536580021, 0.0084934591896779765, 0.010888390921084248] 2018-01-20 01:08:02,081 : INFO : PROGRESS: pass 17, at document #8000/8867 2018-01-20 01:08:07,274 : INFO : optimized alpha [0.029333765653353539, 0.014931441291408491, 0.011063397027131099, 0.013888564744327847, 0.013799335019736634, 0.024541888247133999, 0.010185939185379602, 0.010673194483328289, 0.010551897405979286, 0.013795827595223865, 0.017700001345227895, 0.010926242399832508, 0.015233503199960531, 0.014482306133305808, 0.012321636032501545, 0.016754841778599677, 0.010049845205738196, 0.011900258298661703, 0.014913636775244678, 0.010866340364299133, 0.012134252044676986, 0.018851994624099769, 0.011576922928571693, 0.015079681926010719, 0.012595365284325878, 0.012845990206406085, 0.012144715982868167, 0.01626356885135366, 0.0094923482778546484, 0.016311784674760164, 0.011176253049817066, 0.018755877682832885, 0.0093950126372913164, 0.014054363605318884, 0.010169161943558823, 0.010410734543704085, 0.010873673632204013, 0.02347144309721607, 0.011090250638221515, 0.013065333908115609, 0.016276570177050934, 0.013670841196731088, 0.016122069363248743, 0.011324854755892851, 0.01931023339860484, 0.011030013260851568, 0.017938519575690703, 0.0084989795015676523, 0.0084794937353974646, 0.010883289841858174] 2018-01-20 01:08:07,275 : INFO : PROGRESS: pass 17, at document #8867/8867 2018-01-20 01:08:09,566 : INFO : optimized alpha [0.029527715008151208, 0.015061755245411227, 0.011093749757027789, 0.013888586247644878, 0.013818010713148851, 0.024620934732377385, 0.010206662499522358, 0.01065984699363598, 0.010562593129291059, 0.013849948879153548, 0.01778631394432861, 0.010933809278032834, 0.015281379035797463, 0.014522239616924587, 0.012347800283237533, 0.016798519989672574, 0.010053696217622983, 0.011911198784166617, 0.014980238936346098, 0.010855363624782316, 0.012200010713529347, 0.018899858701229472, 0.011611473944132719, 0.015216146666282777, 0.012589935692269303, 0.01288301810960191, 0.012183273720922011, 0.016298440227917915, 0.0095036011809263039, 0.016352733778305276, 0.01116960998280216, 0.01888816277061587, 0.0093932441606347766, 0.014078243989606781, 0.010224580072003669, 0.010412991619459829, 0.010883990360017056, 0.023596682457951358, 0.011138387376921443, 0.01314243064904899, 0.01629811066696411, 0.013704246000551991, 0.016176556913227971, 0.01135268868929544, 0.019378865716421174, 0.011046888132559765, 0.017984801307650764, 0.0084888843260239946, 0.0085051392966881516, 0.010880410352174541] 2018-01-20 01:08:09,955 : INFO : topic #47 (0.008): 0.039*"ontology" + 0.015*"workshop" + 0.013*"ceur" + 0.011*"proceeding" + 0.010*"language" + 0.008*"arabic" + 0.007*"ceur_workshop" + 0.007*"w" + 0.007*"latency" + 0.006*"measurement" 2018-01-20 01:08:09,956 : INFO : topic #48 (0.009): 0.032*"configuration" + 0.017*"instruction" + 0.015*"simulation" + 0.013*"simulator" + 0.010*"space" + 0.009*"grid" + 0.009*"set" + 0.008*"system" + 0.008*"instruction_set" + 0.008*"embedded" 2018-01-20 01:08:09,957 : INFO : topic #37 (0.024): 0.039*"model" + 0.020*"neural" + 0.013*"data" + 0.012*"network" + 0.012*"learning" + 0.008*"system" + 0.007*"bayesian" + 0.007*"parameter" + 0.007*"using" + 0.007*"gaussian" 2018-01-20 01:08:09,958 : INFO : topic #5 (0.025): 0.022*"system" + 0.020*"web" + 0.016*"service" + 0.009*"user" + 0.009*"application" + 0.008*"design" + 0.008*"technology" + 0.008*"project" + 0.008*"paper" + 0.007*"research" 2018-01-20 01:08:09,959 : INFO : topic #0 (0.030): 0.065*"springer" + 0.054*"berlin" + 0.052*"heidelberg" + 0.030*"berlin_heidelberg" + 0.022*"international" + 0.022*"springer_berlin" + 0.017*"proceeding" + 0.017*"conference" + 0.016*"heidelberg_springer" + 0.010*"verlag" 2018-01-20 01:08:09,961 : INFO : topic diff=0.224062, rho=0.211131 2018-01-20 01:08:10,149 : INFO : PROGRESS: pass 18, at document #2000/8867 2018-01-20 01:08:15,358 : INFO : optimized alpha [0.029578463429521883, 0.015162247226750577, 0.011128326403505273, 0.013886186455989427, 0.013869835973353277, 0.024777897353322455, 0.010232860997092171, 0.010705917884204276, 0.010571052133372651, 0.013868788910015613, 0.017939381251922175, 0.010953736355066412, 0.015292157255952048, 0.014538758893547894, 0.012378967395692515, 0.016882098976904535, 0.0100816511004034, 0.01191234026461725, 0.015026145589435656, 0.010869542815340754, 0.012201092082744591, 0.018915763666714429, 0.011633260237245724, 0.015262840296449852, 0.012661965253382769, 0.012927354284085885, 0.012261625052870439, 0.01638236543013899, 0.0095197775406877985, 0.016403436446291657, 0.01116337625867039, 0.018976039462641234, 0.0093826449382929406, 0.014111930784657353, 0.01024264567750288, 0.010416587693760337, 0.010925240422302208, 0.023724756048473902, 0.011154476681633389, 0.013208789063865238, 0.016367495490768956, 0.013704992028475289, 0.016239906042607197, 0.011368144667124366, 0.01944821542305069, 0.011080549108400693, 0.018056163497130708, 0.0085084884782272718, 0.0085065740039051863, 0.010922894314332044] 2018-01-20 01:08:15,360 : INFO : PROGRESS: pass 18, at document #4000/8867 2018-01-20 01:08:20,619 : INFO : optimized alpha [0.029647519006276282, 0.015251179002087493, 0.011172198808006186, 0.013914699643777543, 0.013918839479589124, 0.024829483223520993, 0.010229566853387963, 0.010730151073986231, 0.010589614355206348, 0.013909005028996569, 0.018016517070796888, 0.010947005758967421, 0.015352212179933971, 0.014548924019633603, 0.0124142911472394, 0.016878482233601309, 0.010078541467236898, 0.011938977939832205, 0.015099108575063597, 0.010892654072845552, 0.012237521270015022, 0.018968850100783346, 0.011668594368401196, 0.015331247239429975, 0.012718887540057377, 0.01296998582922842, 0.012238386308440697, 0.016436958011883201, 0.0095190637690349619, 0.016510605397337424, 0.011189618069133012, 0.019080898157996472, 0.0093854038512190038, 0.014113225166719128, 0.010258917661163609, 0.01042587307195746, 0.010940949715919576, 0.023817810389273882, 0.011186308118062744, 0.013259749395358941, 0.016405390738906549, 0.013729759014399087, 0.01629564216551467, 0.011388301796558248, 0.019526127300256598, 0.011110695736031625, 0.018107417805617389, 0.0085118342540348394, 0.0085216065644925666, 0.010939904988973272] 2018-01-20 01:08:20,621 : INFO : PROGRESS: pass 18, at document #6000/8867 2018-01-20 01:08:25,786 : INFO : optimized alpha [0.029824451161101613, 0.015348877310696776, 0.011207381968763232, 0.013941286028618168, 0.013971715621034693, 0.024979107709210906, 0.010230845114794936, 0.010737144421861305, 0.010608455251244929, 0.013968600449785314, 0.018039582582859592, 0.010955817340406867, 0.015415729773624862, 0.014579988113917782, 0.012439209384010774, 0.016939941322227033, 0.0100811101348892, 0.011964561225290441, 0.015183199550754693, 0.010930863769602962, 0.012271577823112109, 0.019007027071996523, 0.011686481315285444, 0.015371277611388281, 0.012761385244631895, 0.013002476345993455, 0.012248837407675638, 0.016538344824593876, 0.0095266442356446816, 0.016581083827589226, 0.011199012384046246, 0.019156227692357704, 0.0093953807153339987, 0.014119388353396302, 0.010270734322466158, 0.010436670018368092, 0.010955372542388079, 0.023874910568609124, 0.011191877857471855, 0.013287593498125277, 0.01644551217036002, 0.013773589581973717, 0.01634245273030514, 0.011415077386758409, 0.019613742360776683, 0.011127957833485946, 0.018127904257239078, 0.0085163108526780142, 0.0085331130715302021, 0.010942941852447622] 2018-01-20 01:08:25,787 : INFO : PROGRESS: pass 18, at document #8000/8867 2018-01-20 01:08:30,926 : INFO : optimized alpha [0.029924321949139646, 0.015434956275739766, 0.011226408209740229, 0.013959784811101864, 0.014045656915751796, 0.025136257868002273, 0.010245548423679219, 0.010730635561293439, 0.010640944972526321, 0.013996667397460711, 0.018136809576230855, 0.01094502881100249, 0.015467116627678626, 0.014634506960208555, 0.012506586868729446, 0.016963023399465236, 0.010094443401191743, 0.011984036042862562, 0.01524648369854025, 0.010963605813645839, 0.012313630085779986, 0.019119119344469099, 0.011711033241706839, 0.015415082225943453, 0.01279951945558461, 0.013028782120535922, 0.012298519301882804, 0.016587863401150357, 0.0095495279133980104, 0.016666386466390251, 0.011233066780012069, 0.019345527051134209, 0.0094045588077982089, 0.014130888671801223, 0.010276226604420768, 0.010461844565865991, 0.010956022468800151, 0.023992631080208288, 0.011206029498369919, 0.013332452752200205, 0.016457442903249415, 0.013801970829632347, 0.016372357322098328, 0.011409023678287049, 0.019721293215791434, 0.011140626492147773, 0.018210500512694485, 0.0085201147271974677, 0.0085221404460405926, 0.010943706527629852] 2018-01-20 01:08:30,927 : INFO : PROGRESS: pass 18, at document #8867/8867 2018-01-20 01:08:33,269 : INFO : optimized alpha [0.030105525540123444, 0.015560137499939496, 0.011255323603799034, 0.013963155599281954, 0.01405839485323376, 0.025204002187564387, 0.010270854420970178, 0.010714941997035451, 0.010656260070339563, 0.014048633396357995, 0.018221504485487271, 0.010958223567605957, 0.015512206243401598, 0.014680031172650035, 0.012527983139534135, 0.017004192883164123, 0.010095855913076646, 0.011997485527408197, 0.01531657604127493, 0.010957633224271972, 0.012374520077276758, 0.019172931185142274, 0.011741467915115072, 0.015551235294436057, 0.012786048618102114, 0.013070289497717689, 0.012350495246709288, 0.016618736371746299, 0.0095582664800259905, 0.016710905383916438, 0.01122657330144749, 0.019459998577510032, 0.0094077289294357772, 0.014150918893933174, 0.010340361379551138, 0.010461604334727819, 0.010968614689685808, 0.02412093098694185, 0.011258445492470411, 0.013406696954596444, 0.016493086950675293, 0.013837516724355733, 0.016427427934458266, 0.011436173931664313, 0.019784154390029521, 0.011156732390214632, 0.018257848200336577, 0.0085145405079169335, 0.0085473980745635145, 0.010938256013914075] 2018-01-20 01:08:33,667 : INFO : topic #47 (0.009): 0.042*"ontology" + 0.015*"workshop" + 0.013*"ceur" + 0.011*"proceeding" + 0.010*"language" + 0.008*"arabic" + 0.007*"ceur_workshop" + 0.007*"w" + 0.007*"semantic" + 0.007*"latency" 2018-01-20 01:08:33,668 : INFO : topic #48 (0.009): 0.032*"configuration" + 0.018*"instruction" + 0.016*"simulation" + 0.013*"simulator" + 0.011*"space" + 0.009*"grid" + 0.009*"set" + 0.008*"instruction_set" + 0.008*"system" + 0.008*"embedded" 2018-01-20 01:08:33,669 : INFO : topic #37 (0.024): 0.039*"model" + 0.021*"neural" + 0.013*"data" + 0.012*"learning" + 0.012*"network" + 0.008*"parameter" + 0.008*"bayesian" + 0.007*"system" + 0.007*"using" + 0.007*"gaussian" 2018-01-20 01:08:33,670 : INFO : topic #5 (0.025): 0.023*"system" + 0.020*"web" + 0.016*"service" + 0.009*"user" + 0.009*"application" + 0.008*"technology" + 0.008*"design" + 0.008*"project" + 0.008*"paper" + 0.007*"research" 2018-01-20 01:08:33,671 : INFO : topic #0 (0.030): 0.066*"springer" + 0.055*"berlin" + 0.053*"heidelberg" + 0.031*"berlin_heidelberg" + 0.022*"international" + 0.022*"springer_berlin" + 0.017*"proceeding" + 0.017*"conference" + 0.016*"heidelberg_springer" + 0.010*"verlag" 2018-01-20 01:08:33,674 : INFO : topic diff=0.199360, rho=0.206577 2018-01-20 01:08:33,856 : INFO : PROGRESS: pass 19, at document #2000/8867 2018-01-20 01:08:38,961 : INFO : optimized alpha [0.030150613174707713, 0.015661692654578459, 0.011286223234016061, 0.013962238999849424, 0.014109165432237605, 0.025363540324167176, 0.01029872513735049, 0.010761424053634713, 0.010665276258950637, 0.014067072580140148, 0.018378476937559101, 0.010975945722247095, 0.015525418591733679, 0.01469365371661341, 0.012558740422377191, 0.017085009013273939, 0.01012346989279268, 0.012000832646306178, 0.015358526186703927, 0.010976837052653357, 0.012375480103229648, 0.019181456709803316, 0.011765842404789756, 0.015605262560839243, 0.01285995400251984, 0.013112782051568217, 0.01243086579306331, 0.016702092114980308, 0.0095731813914237407, 0.016765727836075689, 0.01122044811200628, 0.019549856198597091, 0.0093976919231317617, 0.014187027945617827, 0.010358659414392023, 0.010468453996370929, 0.011010181854761153, 0.024237438878164801, 0.0112760350580993, 0.013477184619179397, 0.016561869557436874, 0.013834608658236968, 0.016492834393901042, 0.011453534927070932, 0.019853967123236081, 0.011191724572908053, 0.018323792830575038, 0.008534869831237489, 0.0085523441616357585, 0.010976756216008894] 2018-01-20 01:08:38,962 : INFO : PROGRESS: pass 19, at document #4000/8867 2018-01-20 01:08:44,107 : INFO : optimized alpha [0.030210719307827445, 0.01574514101363508, 0.01133098793758989, 0.013990295411911612, 0.01416144536941274, 0.025419910680836884, 0.010298567364962055, 0.010785314943923255, 0.010685386916314016, 0.01411254808299767, 0.018457325100427455, 0.010974281552358097, 0.015576329131377169, 0.014701024220197147, 0.01259493537908893, 0.017083100991997509, 0.010123632190107278, 0.012024356242702216, 0.01542768316898329, 0.011000313147349767, 0.012411626423323496, 0.019230991747355923, 0.011796326550905312, 0.01566970415124767, 0.012917459191102034, 0.013153566810875348, 0.012407760871723822, 0.016754477133047237, 0.0095724549830461126, 0.016873582004599843, 0.011250939429039359, 0.019663000188969937, 0.0094007522279966216, 0.014191246179329684, 0.010374093935186991, 0.010478702232690765, 0.011027676624399536, 0.024321502219875536, 0.01130684891347601, 0.013529586312434344, 0.016597672194820369, 0.013861111430635316, 0.016543923605305089, 0.011471948521468489, 0.019934440965868369, 0.011220940613414072, 0.018373534557240327, 0.0085383460535754033, 0.0085680212955553583, 0.010994684479999468] 2018-01-20 01:08:44,108 : INFO : PROGRESS: pass 19, at document #6000/8867 2018-01-20 01:08:49,298 : INFO : optimized alpha [0.030379135773069495, 0.015840853175309162, 0.011367092134954749, 0.014017919271773337, 0.014214683229327148, 0.025577597454073168, 0.010298633817475275, 0.010793337402008529, 0.010703533200177735, 0.014177262361811075, 0.01847970357793088, 0.01098103117164115, 0.015640457531993824, 0.014727622613784019, 0.012616944124082296, 0.017158059571966831, 0.010126235226346786, 0.012049358940542486, 0.01550488814171249, 0.011038639511110329, 0.012446634514823059, 0.019261763212201615, 0.011816856933179496, 0.015705750673362925, 0.012952629379440327, 0.013187034544331955, 0.012415570042009962, 0.016843271029876091, 0.0095818474896215856, 0.016939556732421319, 0.011257906781375015, 0.019731879115238393, 0.0094156730816590557, 0.014198800015233612, 0.010386224425026333, 0.010490430272773179, 0.011044951661688486, 0.024372709746037287, 0.011316319540764789, 0.013556059187548332, 0.01663222237718379, 0.013910743132578451, 0.016596105122679172, 0.011501657422350155, 0.020030550078555184, 0.011239735821860943, 0.018394579201492248, 0.0085446664236316014, 0.0085828701356300916, 0.010998869123998622] 2018-01-20 01:08:49,299 : INFO : PROGRESS: pass 19, at document #8000/8867 2018-01-20 01:08:54,492 : INFO : optimized alpha [0.030473813650744797, 0.0159295837389208, 0.011384827844859554, 0.014037556775055722, 0.014284464068261074, 0.025732005592176241, 0.0103129871463515, 0.010789167283125076, 0.010736312597889935, 0.014204798788615519, 0.018586314458705389, 0.010971935882460888, 0.01569088279291567, 0.014778972316884506, 0.012682429802361813, 0.017176900102595788, 0.010138420627416805, 0.012070761975178637, 0.015572576826032419, 0.01107499847531662, 0.012494744603158543, 0.019369974881578534, 0.011840357012098911, 0.015752181501870598, 0.012984917153974774, 0.013214282102609456, 0.012466224494345543, 0.016889020660011807, 0.00960434351343494, 0.01701863138872781, 0.011297194155249849, 0.019926467432839847, 0.0094240033692867854, 0.014214450870697856, 0.010389980000716822, 0.010513103493637701, 0.01104636676835972, 0.024494952810848434, 0.011330753404046308, 0.013600968648344041, 0.016639031833196073, 0.013937923568632578, 0.016626507058476576, 0.011490636405918042, 0.020132048617071931, 0.011258553692074832, 0.018462124384755932, 0.008549437697454032, 0.0085711486476212888, 0.010996288543853944] 2018-01-20 01:08:54,494 : INFO : PROGRESS: pass 19, at document #8867/8867 2018-01-20 01:08:56,739 : INFO : optimized alpha [0.030657694002819851, 0.016053053712659957, 0.011415076633119397, 0.014040844535431024, 0.014291357986083848, 0.02578870804182502, 0.01033538845668183, 0.010776280706216097, 0.010756101915837221, 0.014264722324562648, 0.018660429444011053, 0.010990451529846186, 0.0157371958346945, 0.014833498491246709, 0.012708234742384443, 0.017219765912026109, 0.010139862407104147, 0.012080887509091849, 0.015642679890004882, 0.011068506960220508, 0.012553846518846596, 0.01942984517228543, 0.011875388095915138, 0.015880665132153849, 0.012973083964316979, 0.013257142486029994, 0.012519346295033448, 0.016916263020020691, 0.0096105932604288336, 0.017070782986387058, 0.011293391828532237, 0.020037483231252939, 0.0094274224388383818, 0.014240815224946368, 0.010452864188837863, 0.010512903830050074, 0.011060961723280058, 0.024608752838995791, 0.011381932783628543, 0.013678864403113732, 0.016672835997792085, 0.013972222429864067, 0.016674538619619782, 0.011525336930598118, 0.020193829627096666, 0.011279114218610676, 0.018506256324895842, 0.0085421123008534404, 0.0085939920785624586, 0.010996144871593816] 2018-01-20 01:08:57,104 : INFO : topic #47 (0.009): 0.044*"ontology" + 0.016*"workshop" + 0.013*"ceur" + 0.012*"proceeding" + 0.010*"language" + 0.008*"arabic" + 0.008*"semantic" + 0.007*"ceur_workshop" + 0.007*"w" + 0.007*"semantic_web" 2018-01-20 01:08:57,105 : INFO : topic #48 (0.009): 0.032*"configuration" + 0.019*"instruction" + 0.017*"simulation" + 0.014*"simulator" + 0.011*"space" + 0.009*"set" + 0.009*"grid" + 0.008*"instruction_set" + 0.008*"system" + 0.008*"embedded" 2018-01-20 01:08:57,106 : INFO : topic #37 (0.025): 0.039*"model" + 0.021*"neural" + 0.014*"data" + 0.012*"learning" + 0.012*"network" + 0.008*"parameter" + 0.008*"bayesian" + 0.007*"system" + 0.007*"using" + 0.007*"gaussian" 2018-01-20 01:08:57,107 : INFO : topic #5 (0.026): 0.023*"system" + 0.020*"web" + 0.016*"service" + 0.009*"user" + 0.009*"application" + 0.008*"technology" + 0.008*"design" + 0.008*"project" + 0.008*"paper" + 0.007*"research" 2018-01-20 01:08:57,108 : INFO : topic #0 (0.031): 0.067*"springer" + 0.055*"berlin" + 0.053*"heidelberg" + 0.031*"berlin_heidelberg" + 0.023*"international" + 0.022*"springer_berlin" + 0.018*"proceeding" + 0.017*"conference" + 0.016*"heidelberg_springer" + 0.010*"verlag" 2018-01-20 01:08:57,111 : INFO : topic diff=0.179171, rho=0.202305 2018-01-20 01:08:57,291 : INFO : PROGRESS: pass 20, at document #2000/8867 2018-01-20 01:09:02,372 : INFO : optimized alpha [0.030707225446356942, 0.016149312178200814, 0.011448053306495913, 0.014039936210200001, 0.014342638403806863, 0.025940795616678742, 0.010362814085848913, 0.010822026409304263, 0.010762405753394325, 0.01427528082109158, 0.01881566789953475, 0.011011452806038453, 0.015751297279341263, 0.014850332173271041, 0.012736060530837562, 0.017304925347664861, 0.010168187052281353, 0.012080299514692944, 0.015680957635763427, 0.011090283943963219, 0.012554676196664597, 0.019437234612837861, 0.011908269983205828, 0.015935770454665679, 0.013038510382032594, 0.013309919117640932, 0.012603000926169158, 0.017001208479762241, 0.0096252968442259272, 0.017122870563511729, 0.011289569206811003, 0.020131420401519363, 0.0094197263148526368, 0.014276300821795681, 0.010470400112245702, 0.010520759313350588, 0.011101676490471982, 0.024727280040792707, 0.011404435854594532, 0.013747972480030072, 0.016737849716819238, 0.013969773182431871, 0.016743838614266798, 0.011544443350335243, 0.020268161642569289, 0.011314233769015022, 0.018571034885938423, 0.0085639676120887135, 0.0085988856659557147, 0.011039620065783273] 2018-01-20 01:09:02,373 : INFO : PROGRESS: pass 20, at document #4000/8867 2018-01-20 01:09:07,536 : INFO : optimized alpha [0.030758994566724686, 0.016233973370542977, 0.011494790859353633, 0.014071765093797766, 0.014393924061901389, 0.025980941544295387, 0.010363611165718135, 0.010848806245848669, 0.010782896758665178, 0.014325759696363124, 0.018882752211871894, 0.011010044134122604, 0.015799692022982998, 0.014857884580550972, 0.012771930411625252, 0.017302630616495587, 0.010169468162158433, 0.012104576650114364, 0.015743338361078985, 0.011115194284026917, 0.012595571201365041, 0.019481473895660295, 0.011942340412206901, 0.016002871688107993, 0.013091671292151733, 0.013344792336228534, 0.012585044056486534, 0.017055191576066248, 0.0096264898194265581, 0.017231596632619885, 0.011324044173630834, 0.020250918578331827, 0.0094239497938565907, 0.014281701561539668, 0.010491391667280494, 0.010532974831409915, 0.011119706023821204, 0.024808366793678198, 0.011437637352273137, 0.01380317948820425, 0.016763383807306995, 0.013997931993389164, 0.016798769753138005, 0.011562309985561932, 0.020355010429676784, 0.011348152205718364, 0.018617807689465662, 0.0085683832073591565, 0.0086160710456995443, 0.011053852846955389] 2018-01-20 01:09:07,537 : INFO : PROGRESS: pass 20, at document #6000/8867 2018-01-20 01:09:12,614 : INFO : optimized alpha [0.030916087142457142, 0.016334122165924572, 0.011528343848426272, 0.014100265671631435, 0.014447683630049963, 0.026131311979477195, 0.010364624910930556, 0.010859937881124196, 0.010802534315997188, 0.014392580085373887, 0.018899132492085705, 0.011020252442671847, 0.01586288064078293, 0.014883012035631993, 0.012796192618411995, 0.017383892633595546, 0.010176183614645057, 0.012130269659593296, 0.015823384839844114, 0.011153653527557263, 0.012632784179824667, 0.019509205156877884, 0.011964154998421208, 0.016035104806215904, 0.013126486291071599, 0.013379158946047442, 0.012596494096346719, 0.017145537367324014, 0.0096347623689315293, 0.017287961264454272, 0.011331895459781138, 0.020321329188893962, 0.0094407707320724038, 0.014288833502325756, 0.010501697608568051, 0.010541372938534339, 0.01113863880390448, 0.024859283522162219, 0.011445060075389139, 0.01382543538333589, 0.016797976769076475, 0.014046160038512367, 0.016853123778923146, 0.011593603345312335, 0.020447360833307809, 0.01136945897860886, 0.018636077032162447, 0.0085755603536609385, 0.0086324100614463689, 0.011057947602986779] 2018-01-20 01:09:12,615 : INFO : PROGRESS: pass 20, at document #8000/8867 2018-01-20 01:09:17,085 : INFO : optimized alpha [0.030999698699178301, 0.01641905142823526, 0.011545970197398912, 0.014120874942529031, 0.014519365972287524, 0.026281154773910759, 0.010381823052507814, 0.010857911979327553, 0.010839928597918571, 0.014420894729731933, 0.019002261458842654, 0.011018147569901604, 0.015917268008478827, 0.014932734720369377, 0.012857340823772514, 0.017407354887622312, 0.010185156804146086, 0.012148718418363395, 0.015888942721238877, 0.011193594793405107, 0.012680432324662533, 0.019615874999388685, 0.011992555198506108, 0.016084278440835265, 0.013155874625259365, 0.013408722961469631, 0.012644128023574995, 0.017194576327710723, 0.0096607709111526108, 0.017366298899396549, 0.011373945413215982, 0.020514813499525508, 0.0094501305544579649, 0.014305528084051541, 0.010506882670041053, 0.010566897084796563, 0.011145219961470399, 0.024978477708587046, 0.01145511603703063, 0.013867562778818971, 0.016806910219070288, 0.014070762808367106, 0.016883965204699869, 0.01158465572764664, 0.020536773860634728, 0.011386161625299557, 0.018708199830047831, 0.008580391530718429, 0.0086233912440556049, 0.011058703749244119] 2018-01-20 01:09:17,086 : INFO : PROGRESS: pass 20, at document #8867/8867 2018-01-20 01:09:19,264 : INFO : optimized alpha [0.031172959328041844, 0.016556864430816956, 0.011577582097869962, 0.014130551694059232, 0.014530691236458422, 0.026340120835174004, 0.010406205362091019, 0.010845085617870277, 0.010861496628116325, 0.014478539379750391, 0.019079668862973402, 0.011036494633128679, 0.015968506126858482, 0.014971724611197692, 0.012878635064389377, 0.017435143511800571, 0.010193796578192957, 0.012161358958885085, 0.015963055597673369, 0.011194331817228807, 0.012743765237216298, 0.019667875809384432, 0.012017776343078818, 0.01620912894739503, 0.013145813391756774, 0.013449680483346008, 0.012692662153863537, 0.017230411642747228, 0.0096736129872910127, 0.017418197517094471, 0.011369990639366459, 0.020617449300965963, 0.009453688971462185, 0.014331343208403831, 0.010575872289840822, 0.010566725174024802, 0.011164417836286649, 0.025114139367234552, 0.011505133517162182, 0.013939579779080464, 0.016839053707735651, 0.014113607978399757, 0.016948705901765258, 0.011621277333549606, 0.020597558042473645, 0.011405804537978324, 0.01874474038454289, 0.0085772860403150533, 0.0086438764127870112, 0.011063760020070382] 2018-01-20 01:09:19,631 : INFO : topic #47 (0.009): 0.046*"ontology" + 0.016*"workshop" + 0.013*"ceur" + 0.012*"proceeding" + 0.010*"language" + 0.008*"semantic" + 0.008*"arabic" + 0.007*"ceur_workshop" + 0.007*"semantic_web" + 0.007*"w" 2018-01-20 01:09:19,632 : INFO : topic #48 (0.009): 0.032*"configuration" + 0.020*"instruction" + 0.017*"simulation" + 0.014*"simulator" + 0.011*"space" + 0.009*"set" + 0.009*"grid" + 0.009*"instruction_set" + 0.008*"embedded" + 0.008*"system" 2018-01-20 01:09:19,632 : INFO : topic #37 (0.025): 0.039*"model" + 0.021*"neural" + 0.014*"data" + 0.012*"learning" + 0.012*"network" + 0.008*"parameter" + 0.008*"bayesian" + 0.007*"using" + 0.007*"system" + 0.007*"gaussian" 2018-01-20 01:09:19,633 : INFO : topic #5 (0.026): 0.023*"system" + 0.020*"web" + 0.016*"service" + 0.009*"user" + 0.009*"application" + 0.008*"technology" + 0.008*"design" + 0.008*"project" + 0.008*"paper" + 0.008*"research" 2018-01-20 01:09:19,634 : INFO : topic #0 (0.031): 0.068*"springer" + 0.055*"berlin" + 0.053*"heidelberg" + 0.031*"berlin_heidelberg" + 0.023*"international" + 0.022*"springer_berlin" + 0.018*"proceeding" + 0.018*"conference" + 0.016*"heidelberg_springer" + 0.010*"verlag" 2018-01-20 01:09:19,637 : INFO : topic diff=0.162754, rho=0.198288 2018-01-20 01:09:19,812 : INFO : PROGRESS: pass 21, at document #2000/8867 2018-01-20 01:09:24,921 : INFO : optimized alpha [0.031215342295356784, 0.016654295141669634, 0.011610358055995832, 0.014129377055367298, 0.014579525121316335, 0.026463801103585774, 0.01043415517563972, 0.01089223927311523, 0.010870422478878318, 0.014489965019136918, 0.019229610235399673, 0.011059437970279352, 0.015984906992199139, 0.014981551846659957, 0.012906233926135831, 0.017515859950925201, 0.01022271063912268, 0.012165405755062142, 0.016003987165550838, 0.011217352285615378, 0.012750582780170196, 0.019674504025911619, 0.012050621456577947, 0.016263720253967329, 0.013202996262297421, 0.013499262689683066, 0.012773340126410827, 0.017309923482223512, 0.009690827191940905, 0.017462531662284248, 0.011364843671277044, 0.020717635875210489, 0.0094453905806499219, 0.014367527114414675, 0.010595627113041739, 0.010575525385182874, 0.011205330382843777, 0.025224181477691371, 0.011525577107636924, 0.014008836545343354, 0.016905552651796028, 0.014110186689433259, 0.017008043271207897, 0.011637463651263511, 0.020672444161332147, 0.011440929352623265, 0.018808361137381362, 0.0086014108673931004, 0.0086554308305708486, 0.011105401397098954] 2018-01-20 01:09:24,923 : INFO : PROGRESS: pass 21, at document #4000/8867 2018-01-20 01:09:29,950 : INFO : optimized alpha [0.031266347862495311, 0.016741516445233272, 0.011655676472393533, 0.014157734831918158, 0.014632691494445847, 0.026499922794560506, 0.010436835326541929, 0.010919618390842845, 0.010888032340858734, 0.014528353658102294, 0.019298770073559512, 0.011059201674090437, 0.016030767838186483, 0.014991193890874766, 0.012938014265120146, 0.017516798062976503, 0.010225996106533446, 0.012190324821505647, 0.016067456928968906, 0.011242418934544921, 0.01278970462241282, 0.019719849192944085, 0.012083499772899383, 0.016327156733047372, 0.013253238787057255, 0.013540355635320354, 0.012760515054426489, 0.017365401397207235, 0.009691890292990132, 0.017572390803696363, 0.011396483602352238, 0.02084564142132744, 0.0094506813077234759, 0.014375461525801631, 0.010611543696675467, 0.010587577465736436, 0.011226000397898648, 0.025307207500165244, 0.011561264278395753, 0.014065560714280737, 0.016929476438011494, 0.014135749052836957, 0.017061655452943233, 0.011654761724595369, 0.020751687943463843, 0.011478331583000557, 0.018858086752838982, 0.0086058306867707341, 0.0086739999173336257, 0.011123721379467407] 2018-01-20 01:09:29,951 : INFO : PROGRESS: pass 21, at document #6000/8867 2018-01-20 01:09:35,011 : INFO : optimized alpha [0.031419578333894124, 0.016836076126269309, 0.011690192076031966, 0.01418845842752329, 0.014683963660254666, 0.026643953455192333, 0.010439699956959888, 0.010935863699437909, 0.010908091233233486, 0.014594537884608103, 0.019317185276327142, 0.011068265781348754, 0.016097779788118743, 0.01501822252010167, 0.012963381661487479, 0.017591937293853849, 0.010226522159850848, 0.012217818693512184, 0.016155173480914477, 0.011275393272679162, 0.01283147055722701, 0.019746780545696473, 0.012107799414920142, 0.016358956884921994, 0.013285256748069576, 0.013574158781003562, 0.012774302895765581, 0.017451934219043307, 0.0097018068211463197, 0.017629711444690482, 0.01140857087181354, 0.020917696017333513, 0.009471163029272334, 0.014382276700292717, 0.010622100463024121, 0.010601058170938886, 0.011239781289373092, 0.02535983889080318, 0.011570215428552194, 0.014086531257371913, 0.016967476568458865, 0.014178421717133126, 0.017114747792670151, 0.011684155199780768, 0.020844630428394764, 0.01149750781713535, 0.018869828220976857, 0.0086137990620182067, 0.008690904720755132, 0.011126516409493907] 2018-01-20 01:09:35,012 : INFO : PROGRESS: pass 21, at document #8000/8867 2018-01-20 01:09:40,113 : INFO : optimized alpha [0.031499662961057877, 0.0169205509146561, 0.011706564262382618, 0.014209982761220865, 0.014751753242474162, 0.026782188398575841, 0.010454433480610327, 0.010934761911461905, 0.010942456500338807, 0.014625281631899774, 0.019430233764621756, 0.011066270644640079, 0.016156077586302244, 0.015071030422339628, 0.013025464110363004, 0.01761866639426549, 0.010239427253008188, 0.012240624016012792, 0.016217073056083824, 0.011309899229248721, 0.012874775758421461, 0.019854178026662613, 0.012138643411553303, 0.016414275267309521, 0.013315892952015797, 0.013604666209378584, 0.012824161023988223, 0.017504413914853134, 0.0097273292884902697, 0.017712684481511719, 0.011449923017873339, 0.021120766993821066, 0.0094796078729190245, 0.014401352370285241, 0.010626529795134928, 0.010630390203257758, 0.011246926006635159, 0.02547078708598309, 0.011588634660307597, 0.01413705558320941, 0.016973328832738346, 0.014200567676790945, 0.01714779172762048, 0.011674935078841752, 0.020949342615833581, 0.011509964294401095, 0.01894465898286362, 0.0086228222936617846, 0.0086836202379817137, 0.011131540879087595] 2018-01-20 01:09:40,114 : INFO : PROGRESS: pass 21, at document #8867/8867 2018-01-20 01:09:42,241 : INFO : optimized alpha [0.031663240448285487, 0.017057152599875794, 0.011734177610808074, 0.014216136507421715, 0.014760831665115262, 0.026851142722639887, 0.010480725631012079, 0.010924320917765382, 0.010965842716624815, 0.014687691806309223, 0.019519628614411328, 0.011086926535595419, 0.016208422799187543, 0.015105245083317349, 0.013042408236015407, 0.017640175355197931, 0.010245607171087686, 0.012258354081981669, 0.016291028940288867, 0.011307477841744101, 0.012942128898753397, 0.019903108134946369, 0.012159893021012705, 0.01653531360905388, 0.013304708210062275, 0.013643775665300834, 0.012871106781485472, 0.017548644658839027, 0.0097398262593484317, 0.01776034291202434, 0.011445811816454855, 0.021214536485205614, 0.0094854280740881234, 0.014429792618858414, 0.010696671040587916, 0.010627691808990142, 0.01126798328844723, 0.025616235268278643, 0.011647989270217133, 0.014209562986036631, 0.017003877431931456, 0.014235812790889875, 0.017229126687615861, 0.011713471474282026, 0.021004420967044447, 0.011533939442479466, 0.018970047664723575, 0.0086256159999802788, 0.0087036962501316397, 0.011138965547769952] 2018-01-20 01:09:42,601 : INFO : topic #47 (0.009): 0.047*"ontology" + 0.016*"workshop" + 0.013*"ceur" + 0.012*"proceeding" + 0.010*"language" + 0.009*"semantic" + 0.008*"semantic_web" + 0.008*"arabic" + 0.007*"ceur_workshop" + 0.007*"w" 2018-01-20 01:09:42,603 : INFO : topic #48 (0.009): 0.032*"configuration" + 0.021*"instruction" + 0.018*"simulation" + 0.014*"simulator" + 0.012*"space" + 0.010*"set" + 0.009*"grid" + 0.009*"instruction_set" + 0.009*"embedded" + 0.008*"system" 2018-01-20 01:09:42,605 : INFO : topic #37 (0.026): 0.040*"model" + 0.021*"neural" + 0.014*"data" + 0.013*"learning" + 0.012*"network" + 0.008*"parameter" + 0.008*"bayesian" + 0.007*"using" + 0.007*"gaussian" + 0.007*"system" 2018-01-20 01:09:42,606 : INFO : topic #5 (0.027): 0.023*"system" + 0.020*"web" + 0.016*"service" + 0.009*"application" + 0.009*"user" + 0.009*"technology" + 0.008*"design" + 0.008*"project" + 0.008*"paper" + 0.008*"research" 2018-01-20 01:09:42,608 : INFO : topic #0 (0.032): 0.069*"springer" + 0.056*"berlin" + 0.054*"heidelberg" + 0.031*"berlin_heidelberg" + 0.023*"international" + 0.023*"springer_berlin" + 0.018*"proceeding" + 0.018*"conference" + 0.016*"heidelberg_springer" + 0.011*"verlag" 2018-01-20 01:09:42,611 : INFO : topic diff=0.149585, rho=0.194501 2018-01-20 01:09:42,787 : INFO : PROGRESS: pass 22, at document #2000/8867 2018-01-20 01:09:47,693 : INFO : optimized alpha [0.031696279517737888, 0.017155779179385246, 0.011764553882191445, 0.014218924803253929, 0.014808792751024367, 0.026976504131794171, 0.010508192361559027, 0.010970686957039396, 0.01097524385315995, 0.014701479496396818, 0.019667989926202802, 0.011108548539991594, 0.016227231502452897, 0.015117072527487492, 0.013074943450447062, 0.017721855584513016, 0.01027713241623217, 0.012266759266615109, 0.016336316153277179, 0.01133411525547662, 0.012949851026643063, 0.019905076564956957, 0.01219398955774577, 0.016589432886609717, 0.01335910003428877, 0.01369687602544737, 0.012952571389927026, 0.017629526817247738, 0.0097548112790017509, 0.017807542678794783, 0.011440548138392306, 0.021308434628172671, 0.0094792468523696872, 0.014470876797096697, 0.010716671500605263, 0.010636327426328871, 0.011309141994312403, 0.02572311614839989, 0.011669654014512037, 0.014277610288860067, 0.017068558183900285, 0.014235912750399721, 0.017296954469580395, 0.01173137946634986, 0.021077737030920397, 0.011572547470941406, 0.019034722597369207, 0.008647734831126682, 0.0087158085229887168, 0.011178851183460441] 2018-01-20 01:09:47,694 : INFO : PROGRESS: pass 22, at document #4000/8867 2018-01-20 01:09:52,682 : INFO : optimized alpha [0.031734957656134676, 0.017250733813127833, 0.011809685349310747, 0.014246765985166434, 0.014856819431310814, 0.027003456539721315, 0.010512687345660354, 0.010998589169715856, 0.01099639776985138, 0.014742048516645291, 0.019731385287400847, 0.011110555086658414, 0.016275166592731247, 0.015128671817968656, 0.013107770989141919, 0.017720954893013532, 0.010282335467433595, 0.012294512784985806, 0.016399330888878169, 0.011360571292782005, 0.012983464613126994, 0.019940068840698273, 0.012229198578800265, 0.016655715808073172, 0.013410471534074615, 0.013734840728748827, 0.012935914779139891, 0.017681193783677045, 0.0097557315708019197, 0.017915042640086604, 0.011476050786510232, 0.021438992819625146, 0.0094863940623049984, 0.01447835036746844, 0.010734829647096355, 0.010647134807737433, 0.011329124227129717, 0.02580306773382382, 0.011699650489354006, 0.014331594534036133, 0.01709436822557877, 0.014267373076712084, 0.017350517245626609, 0.01175037230993335, 0.02115154330227911, 0.011617822062567489, 0.019081803262157804, 0.0086537406682758184, 0.008734867543937392, 0.011195699856717275] 2018-01-20 01:09:52,683 : INFO : PROGRESS: pass 22, at document #6000/8867 2018-01-20 01:09:57,700 : INFO : optimized alpha [0.031888380386062343, 0.017344634343419069, 0.011848625703710643, 0.014276902623354395, 0.014898653218148344, 0.027152204665227886, 0.010516312629032114, 0.011019771685239845, 0.011014641219993571, 0.014806126123838075, 0.019743712595228245, 0.011121700388140964, 0.016341131728672804, 0.015151660166168958, 0.013135548449831487, 0.017795444648438869, 0.010290741194230887, 0.012316504807411055, 0.016488278198558345, 0.011399215266904244, 0.013021020105916594, 0.019964684660035564, 0.012254751917113026, 0.01668861142579272, 0.013442491991791365, 0.013768175418219659, 0.012949551906410802, 0.017758816859363199, 0.0097654184556850727, 0.017964695289743204, 0.011485574633725279, 0.021512938468299176, 0.0095048791193640316, 0.014486111144534485, 0.010745614758209157, 0.010658336045782977, 0.011342264351737946, 0.025855091610108363, 0.011714461388910699, 0.014353943995664786, 0.017135872105803616, 0.014308846914117039, 0.017405282406194905, 0.011779027652837231, 0.021243200150414691, 0.011642704620592786, 0.019094916969244931, 0.0086615971201023396, 0.0087514455189434213, 0.011199393738410281] 2018-01-20 01:09:57,701 : INFO : PROGRESS: pass 22, at document #8000/8867 2018-01-20 01:10:02,707 : INFO : optimized alpha [0.03196572308519003, 0.01743334761249736, 0.011864928371123148, 0.014297906006542814, 0.014954143283521418, 0.027289792413387588, 0.010532686643659892, 0.011019461152533938, 0.01104710391562575, 0.014836323783910482, 0.019851466405144704, 0.011121927277573499, 0.016399948030177809, 0.01520745045153991, 0.013203630012638825, 0.017821770249803259, 0.010302385933124989, 0.012337428757274317, 0.016554496862009362, 0.011433869794465945, 0.013065216264521626, 0.020073120002343084, 0.012283212956567487, 0.016743499144110861, 0.013473136492964551, 0.013796932827044639, 0.013006642161299218, 0.017804199662380012, 0.0097933212235777646, 0.018050661347921774, 0.011527370929055456, 0.02171506801342293, 0.0095160860335283225, 0.014505999359711634, 0.010750346093144047, 0.010685205995819194, 0.011356529044563624, 0.025963014442987212, 0.011731836410632064, 0.01439909520579819, 0.017145303079098159, 0.014335627389735052, 0.017445193868608447, 0.011772967065464341, 0.021342399405837913, 0.011657489024828702, 0.019157742133541676, 0.0086713017085055435, 0.0087466311361775132, 0.011206394539345165] 2018-01-20 01:10:02,708 : INFO : PROGRESS: pass 22, at document #8867/8867 2018-01-20 01:10:04,953 : INFO : optimized alpha [0.032142187344296309, 0.017580414685679217, 0.011893901197703049, 0.014310090730727917, 0.014954648566804877, 0.027337134766201858, 0.01055610898683907, 0.011008762451338541, 0.011074746699632284, 0.014899872760539728, 0.01993532260273442, 0.011142274266455712, 0.016449625440797328, 0.015243859933681905, 0.013225003293393362, 0.017841123037662637, 0.010306087277799551, 0.012354600665933524, 0.01663230275039073, 0.011435829009836279, 0.013130822068390354, 0.020124138811904287, 0.012308795439749392, 0.016860915262916901, 0.01346688155227193, 0.013837396932813995, 0.013051748931218637, 0.017849016540602985, 0.0098055209663829301, 0.018094089265461705, 0.011525613960788486, 0.021805160556069056, 0.0095198266877163651, 0.014533727691013229, 0.010824122931217267, 0.01069207617650942, 0.011374332192479316, 0.026100542679728174, 0.011789933313199309, 0.014472148467541539, 0.017170396854719626, 0.014372978278288448, 0.017515237273211665, 0.011815913271570822, 0.021387247141375989, 0.01168554963312598, 0.019198258773728518, 0.0086721261254109894, 0.0087682802415570433, 0.011218519467058451] 2018-01-20 01:10:05,305 : INFO : topic #47 (0.009): 0.049*"ontology" + 0.017*"workshop" + 0.013*"ceur" + 0.012*"proceeding" + 0.010*"semantic" + 0.010*"language" + 0.008*"semantic_web" + 0.007*"ceur_workshop" + 0.007*"w" + 0.007*"arabic" 2018-01-20 01:10:05,306 : INFO : topic #48 (0.009): 0.032*"configuration" + 0.022*"instruction" + 0.019*"simulation" + 0.015*"simulator" + 0.012*"space" + 0.010*"set" + 0.009*"grid" + 0.009*"instruction_set" + 0.009*"embedded" + 0.008*"system" 2018-01-20 01:10:05,307 : INFO : topic #37 (0.026): 0.040*"model" + 0.021*"neural" + 0.014*"data" + 0.013*"learning" + 0.012*"network" + 0.008*"parameter" + 0.008*"bayesian" + 0.007*"gaussian" + 0.007*"using" + 0.007*"system" 2018-01-20 01:10:05,308 : INFO : topic #5 (0.027): 0.023*"system" + 0.020*"web" + 0.016*"service" + 0.009*"application" + 0.009*"user" + 0.009*"technology" + 0.008*"project" + 0.008*"design" + 0.008*"paper" + 0.008*"research" 2018-01-20 01:10:05,309 : INFO : topic #0 (0.032): 0.069*"springer" + 0.056*"berlin" + 0.054*"heidelberg" + 0.031*"berlin_heidelberg" + 0.023*"international" + 0.023*"springer_berlin" + 0.018*"proceeding" + 0.018*"conference" + 0.016*"heidelberg_springer" + 0.011*"verlag" 2018-01-20 01:10:05,312 : INFO : topic diff=0.139049, rho=0.190924 2018-01-20 01:10:05,484 : INFO : PROGRESS: pass 23, at document #2000/8867 2018-01-20 01:10:10,435 : INFO : optimized alpha [0.032172236686388224, 0.017674934529649185, 0.011921874656385643, 0.014313908572567487, 0.015000756140681391, 0.02745686739474857, 0.010585092130008272, 0.011058597123254155, 0.011089799386275292, 0.014910282887383686, 0.020082385367109361, 0.011168950328067818, 0.016461159454418804, 0.015256119948735337, 0.01325348755994527, 0.017925656444645145, 0.010335173908576552, 0.01236137307869966, 0.016677072801917527, 0.011460447578480512, 0.013139493878681789, 0.020127664246515766, 0.012344045493589494, 0.016917881228277969, 0.013521041739613531, 0.013889994014930792, 0.013132757583925981, 0.017931575357252049, 0.009820203134944366, 0.018140765826003009, 0.011520143349620876, 0.021892754810722864, 0.0095111215233601061, 0.01457534920338443, 0.010847341867305947, 0.01070254501822223, 0.011411354825495761, 0.026215092449087651, 0.011816217433082294, 0.014536238752857675, 0.017234935124042092, 0.014370828364725415, 0.017581524320128643, 0.011839907823205684, 0.021459600918671495, 0.011724036187238588, 0.019260341829783395, 0.0086964235889333569, 0.0087801023911450327, 0.011255597564058228] 2018-01-20 01:10:10,436 : INFO : PROGRESS: pass 23, at document #4000/8867 2018-01-20 01:10:15,335 : INFO : optimized alpha [0.032205232430838997, 0.017769224106922152, 0.011966847375177701, 0.014345225451072922, 0.015051275721849776, 0.027488966436668184, 0.01059132401958569, 0.011090095962595782, 0.011108079857314533, 0.01494749270177111, 0.020148129557647596, 0.011170872906235913, 0.016506612114786316, 0.015262384861885735, 0.013288575315393474, 0.01792625833099154, 0.010344164024341988, 0.01238731056216918, 0.016734651875379567, 0.011487100882607312, 0.013172683449092263, 0.020165852414024004, 0.012378023811477868, 0.016982114629123444, 0.013574802115297595, 0.013926166667223057, 0.013117237147302263, 0.017981404601115282, 0.0098209555368205925, 0.018247912668110034, 0.01155170455863056, 0.022015383443101572, 0.0095129015250561347, 0.014579505140883608, 0.010867704515751336, 0.010715110318963845, 0.011429559426300775, 0.026289534238501305, 0.011847322389096186, 0.014586341162788861, 0.017255937358772226, 0.014406705343234665, 0.017633553349763163, 0.011855929928635021, 0.021528190094264325, 0.011769176307258263, 0.019304930048203401, 0.008703137529553602, 0.0087979532756524911, 0.011274181089177309] 2018-01-20 01:10:15,336 : INFO : PROGRESS: pass 23, at document #6000/8867 2018-01-20 01:10:20,490 : INFO : optimized alpha [0.032359213216678603, 0.017871096251176426, 0.012003425792681859, 0.014374652191476382, 0.015098458136802323, 0.027624177312562767, 0.010596690374879899, 0.011108617257632607, 0.011126659572295312, 0.015008124693059665, 0.020160535045654116, 0.011188185928278534, 0.016570112855542871, 0.015287230067533665, 0.013316023575523242, 0.018003495572933577, 0.010352378151566444, 0.012411089474824322, 0.016818486249975603, 0.011525843815598345, 0.013214914021311775, 0.020188007839223133, 0.012408385188730003, 0.017012899233051278, 0.013609313708056212, 0.013957903653675018, 0.01312437091773682, 0.018058931849012898, 0.0098331570398471676, 0.018297104356698232, 0.011564199537402455, 0.022104229259415888, 0.0095348892878770377, 0.014592373486442848, 0.010878636609286787, 0.010726026113236921, 0.011443160840241766, 0.026333882517153382, 0.011861135832339486, 0.014604836348078635, 0.017304185471181212, 0.014448353570955719, 0.017693421637937974, 0.01188267957172221, 0.021614751857812701, 0.011795168528281852, 0.019314006101675728, 0.0087116637287477355, 0.0088183577639316599, 0.011281911271826718] 2018-01-20 01:10:20,491 : INFO : PROGRESS: pass 23, at document #8000/8867 2018-01-20 01:10:25,611 : INFO : optimized alpha [0.03243413308911116, 0.01795387341238857, 0.012020820031251447, 0.014399159001520433, 0.015152017236177871, 0.02775131392759678, 0.010612641537113379, 0.01111009604366831, 0.011160411116861044, 0.015033562816690075, 0.02028241397439156, 0.011187195599794164, 0.01662497667551293, 0.015340266778107393, 0.013383727475807884, 0.018032931284775334, 0.0103686905667742, 0.012433799845085971, 0.016882722180359234, 0.011566116287065139, 0.013254936777867213, 0.02029736070315958, 0.01243565542613408, 0.017067451089938589, 0.013632026877560944, 0.013986340574271276, 0.013177374001317653, 0.018107846420825295, 0.0098633507637673895, 0.018388009512651467, 0.011604226157588543, 0.022303299409542964, 0.0095479016857513855, 0.014611640892687159, 0.010888716132199787, 0.010752471073789632, 0.011454697450031498, 0.026441801455167453, 0.011877501668335268, 0.014651739072754471, 0.017317278056772852, 0.014468414568078397, 0.017736898814523241, 0.011878436070683654, 0.021723664207757142, 0.011813344565532643, 0.019385317220044235, 0.0087212122750584785, 0.0088117395768846103, 0.0112907209004305] 2018-01-20 01:10:25,612 : INFO : PROGRESS: pass 23, at document #8867/8867 2018-01-20 01:10:27,751 : INFO : optimized alpha [0.032594572799653693, 0.018099693356245689, 0.012051167092169449, 0.014407559922174133, 0.015147455305296305, 0.027797187309645546, 0.010640178479762582, 0.011086957049336, 0.011192063583789493, 0.015098508870235079, 0.020369674265543106, 0.011209642862580783, 0.016664918627003988, 0.015378825523339421, 0.013400579400298245, 0.018058076576266329, 0.010376735982403074, 0.012447710951568011, 0.016960703690307988, 0.011564753489625285, 0.013327483643848453, 0.020345800340065737, 0.012462835211024507, 0.017181343290017605, 0.013627633166837487, 0.014025009097342741, 0.013223959656613356, 0.01815718910432711, 0.0098752083078113735, 0.018427276534646767, 0.011599750778217559, 0.022394552726249697, 0.009551719826208149, 0.014641775805457519, 0.010958828177208522, 0.010756780801107419, 0.011471872149303822, 0.026571492957091151, 0.011934385236210533, 0.014728694061801243, 0.017336766318177599, 0.014504631453444477, 0.017815367230659497, 0.011923110702557641, 0.021767912931774502, 0.011837590615691939, 0.019418999868370923, 0.0087238590284122643, 0.0088329842196844845, 0.011299974898053631] 2018-01-20 01:10:28,108 : INFO : topic #47 (0.009): 0.051*"ontology" + 0.017*"workshop" + 0.013*"ceur" + 0.012*"proceeding" + 0.011*"semantic" + 0.010*"language" + 0.008*"semantic_web" + 0.007*"ceur_workshop" + 0.007*"w" + 0.007*"arabic" 2018-01-20 01:10:28,109 : INFO : topic #48 (0.009): 0.032*"configuration" + 0.023*"instruction" + 0.020*"simulation" + 0.015*"simulator" + 0.012*"space" + 0.010*"set" + 0.009*"embedded" + 0.009*"instruction_set" + 0.009*"grid" + 0.008*"system" 2018-01-20 01:10:28,110 : INFO : topic #37 (0.027): 0.040*"model" + 0.021*"neural" + 0.014*"data" + 0.013*"learning" + 0.012*"network" + 0.008*"parameter" + 0.008*"bayesian" + 0.007*"gaussian" + 0.007*"using" + 0.007*"system" 2018-01-20 01:10:28,111 : INFO : topic #5 (0.028): 0.023*"system" + 0.021*"web" + 0.017*"service" + 0.009*"application" + 0.009*"user" + 0.009*"technology" + 0.008*"project" + 0.008*"design" + 0.008*"paper" + 0.008*"research" 2018-01-20 01:10:28,112 : INFO : topic #0 (0.033): 0.070*"springer" + 0.056*"berlin" + 0.054*"heidelberg" + 0.032*"berlin_heidelberg" + 0.024*"international" + 0.023*"springer_berlin" + 0.019*"proceeding" + 0.018*"conference" + 0.016*"heidelberg_springer" + 0.011*"verlag" 2018-01-20 01:10:28,115 : INFO : topic diff=0.130641, rho=0.187536 2018-01-20 01:10:28,287 : INFO : PROGRESS: pass 24, at document #2000/8867 2018-01-20 01:10:33,196 : INFO : optimized alpha [0.032625691042406854, 0.018193449758867303, 0.012077921916299883, 0.014409604657750158, 0.015197424246972634, 0.027906359182673673, 0.010667588319158044, 0.011135002559955347, 0.011204303984788454, 0.015108508506782335, 0.020515313808414259, 0.011234811595994711, 0.016677455596600031, 0.015388644397619137, 0.013428833451778418, 0.018140031633547642, 0.010407231061534001, 0.01245638481918049, 0.017005009103376639, 0.011589557012677505, 0.013335770457072401, 0.020343354450285799, 0.012499158985723454, 0.017239668625301463, 0.01367782970888976, 0.014078504341215372, 0.013300870243332971, 0.018236112633983587, 0.0098904397874001928, 0.018473330113139175, 0.011597357805632756, 0.02247583084556903, 0.0095459236560295532, 0.014682503630985279, 0.010984152779756826, 0.01076800709779349, 0.011514546223514945, 0.02667324288586732, 0.011964112539012934, 0.014791670591640545, 0.017401175493845244, 0.014496266322879599, 0.017878247972495962, 0.011947438318111497, 0.021837256868503824, 0.011875931459728003, 0.019480314674012473, 0.0087461738073486405, 0.0088444971876017429, 0.011334298933590056] 2018-01-20 01:10:33,198 : INFO : PROGRESS: pass 24, at document #4000/8867 2018-01-20 01:10:38,060 : INFO : optimized alpha [0.032647409803444043, 0.018276953839723425, 0.012119375797245759, 0.014444272418677493, 0.015244671188671923, 0.027933627536992144, 0.010673500959948859, 0.011165956383793562, 0.011226043465965037, 0.015156667635341077, 0.020575418526525811, 0.011240776264863342, 0.016720607003487441, 0.015393986359314488, 0.013463603960728199, 0.01814374781524819, 0.01041494730042908, 0.012484020265750335, 0.017062204059153846, 0.011620698202744195, 0.013371087314142403, 0.020381178121869126, 0.012529561464558813, 0.017298653075047063, 0.013733938773718925, 0.014113048944217567, 0.013286669130855864, 0.018292857440882545, 0.0098918860076309956, 0.018578411668424138, 0.011632692145391459, 0.022601155167424176, 0.0095522095081206973, 0.014693044738225858, 0.010999454773109494, 0.010781283650007851, 0.011534268314437253, 0.026755700417380318, 0.011996362409791007, 0.014837916473879127, 0.017419119789460879, 0.014536604822321423, 0.017933578425731622, 0.011966153217733371, 0.021900757569789819, 0.011918771707775147, 0.019524352215116333, 0.0087535531245094868, 0.00886199681264376, 0.011355606618761735] 2018-01-20 01:10:38,061 : INFO : PROGRESS: pass 24, at document #6000/8867 2018-01-20 01:10:43,182 : INFO : optimized alpha [0.03279958502307407, 0.018367975904865701, 0.012158112366920592, 0.014472976265396984, 0.015285732863454667, 0.0280855694678897, 0.010678544722750155, 0.011187195875879146, 0.011244901310896635, 0.015222411397363194, 0.020591668483976984, 0.011257721239941723, 0.016780173769246481, 0.015412032448053477, 0.013492089561758154, 0.018223744487865453, 0.010423866488254009, 0.012510680188641295, 0.017145705615258369, 0.011658388125156668, 0.013412893034016753, 0.020399158071637533, 0.01255758191459681, 0.017327559006683955, 0.01376836228494779, 0.014144390376596157, 0.013297457746419697, 0.018366903337474771, 0.0099010117688160953, 0.018627107623424292, 0.011643719406143678, 0.022683871278250022, 0.0095722174894599301, 0.014706648660599163, 0.011012611956910662, 0.010795961042572832, 0.011550480438034704, 0.026800115994096717, 0.01200686922329941, 0.014855330141890118, 0.017465750345688837, 0.0145771742103297, 0.017998536361490056, 0.01199210517400935, 0.021984483950581595, 0.011944738556672834, 0.019534934449978312, 0.0087619226788784171, 0.0088836926239946202, 0.011365107201076747] 2018-01-20 01:10:43,184 : INFO : PROGRESS: pass 24, at document #8000/8867 2018-01-20 01:10:48,264 : INFO : optimized alpha [0.032872593443430222, 0.018453600649365989, 0.012175509812095375, 0.014492822240232682, 0.015337480867541638, 0.028210550239387459, 0.010694053993778116, 0.011192635563710402, 0.01128194623022878, 0.015254455059778266, 0.020704323739782138, 0.011257645736011025, 0.016842366693097726, 0.015466726752640552, 0.013560676708896069, 0.018247426914065792, 0.010440816835365933, 0.012531568463930578, 0.017207965527052876, 0.011700890183365655, 0.013452425525273638, 0.020500021911202319, 0.012584889435974538, 0.017385208597811421, 0.013792285959274184, 0.014173839996796441, 0.013353878382786965, 0.018419124452502714, 0.0099307325507335571, 0.018714132648455981, 0.011681959592503528, 0.022875783676124081, 0.0095851269682555515, 0.014726589112103656, 0.011020773901121373, 0.010827989775738678, 0.011564600056682597, 0.026890173113372797, 0.012025563203166011, 0.01489842128757247, 0.01747920115982856, 0.014597679148534938, 0.01803854185123499, 0.011986190586145768, 0.022092500079662038, 0.011964051648668213, 0.019609201393780095, 0.0087696631523956139, 0.0088769386603570321, 0.011371427099896371] 2018-01-20 01:10:48,265 : INFO : PROGRESS: pass 24, at document #8867/8867 2018-01-20 01:10:50,393 : INFO : optimized alpha [0.033039513357889266, 0.018586084515844658, 0.012199434414505567, 0.01450398079689314, 0.01533806617546545, 0.028248687112988197, 0.010718755075081372, 0.011176663907262831, 0.011315143436299252, 0.015317181834664444, 0.020786068667324277, 0.011277201066875646, 0.01688008315385349, 0.015500921176309029, 0.013578980607798254, 0.018266351385339677, 0.010453059579516404, 0.012544908938667212, 0.017282396536889569, 0.011703712975437435, 0.01352319655669177, 0.020555125666125752, 0.012602754259840756, 0.017503180365294396, 0.013780714752730034, 0.014213932383171788, 0.013390224670362437, 0.018465002660600815, 0.0099422638761911403, 0.01875366670078181, 0.01167728004070441, 0.022968674704914635, 0.0095889849818603344, 0.014759052336472612, 0.011089740524038043, 0.01083664363235283, 0.011581090347171774, 0.027024885661632289, 0.012086398408831148, 0.014969815849109593, 0.017501069589061464, 0.014639118629548404, 0.018109576626616601, 0.012032523516780096, 0.022136724448379438, 0.011994920357299146, 0.019648909226297428, 0.0087703589271157407, 0.0088997058596248669, 0.011385232811435064] 2018-01-20 01:10:50,744 : INFO : topic #47 (0.009): 0.052*"ontology" + 0.017*"workshop" + 0.014*"ceur" + 0.013*"proceeding" + 0.011*"semantic" + 0.010*"language" + 0.009*"semantic_web" + 0.008*"ceur_workshop" + 0.008*"w" + 0.007*"arabic" 2018-01-20 01:10:50,745 : INFO : topic #48 (0.009): 0.032*"configuration" + 0.024*"instruction" + 0.021*"simulation" + 0.015*"simulator" + 0.013*"space" + 0.010*"set" + 0.009*"embedded" + 0.009*"instruction_set" + 0.009*"grid" + 0.009*"system" 2018-01-20 01:10:50,746 : INFO : topic #37 (0.027): 0.040*"model" + 0.021*"neural" + 0.014*"data" + 0.013*"learning" + 0.012*"network" + 0.008*"parameter" + 0.008*"bayesian" + 0.007*"gaussian" + 0.007*"using" + 0.007*"neural_network" 2018-01-20 01:10:50,747 : INFO : topic #5 (0.028): 0.024*"system" + 0.021*"web" + 0.017*"service" + 0.009*"application" + 0.009*"technology" + 0.009*"user" + 0.008*"project" + 0.008*"design" + 0.008*"paper" + 0.008*"research" 2018-01-20 01:10:50,748 : INFO : topic #0 (0.033): 0.071*"springer" + 0.057*"berlin" + 0.055*"heidelberg" + 0.032*"berlin_heidelberg" + 0.024*"international" + 0.023*"springer_berlin" + 0.019*"proceeding" + 0.018*"conference" + 0.016*"heidelberg_springer" + 0.011*"verlag" 2018-01-20 01:10:50,751 : INFO : topic diff=0.123994, rho=0.184323 2018-01-20 01:10:50,917 : INFO : PROGRESS: pass 25, at document #2000/8867 2018-01-20 01:10:55,920 : INFO : optimized alpha [0.03306883230058548, 0.018681271972260487, 0.012226201701317487, 0.014509718476732608, 0.015383433649381682, 0.028355814245254334, 0.010745653205994882, 0.011224936285010796, 0.011328664043549182, 0.015333621180574275, 0.020926580442272368, 0.011305060962898661, 0.016898448855257674, 0.015511423435850834, 0.013604525506701571, 0.018342552441513251, 0.010485911835802712, 0.012553093396668637, 0.017326360061060725, 0.011731846030739247, 0.013537364112802935, 0.020546761796018349, 0.012640322110337927, 0.017557882463935463, 0.013833328936937422, 0.01426175008109235, 0.013465659464969951, 0.018540439138340532, 0.0099562510245207996, 0.018797577352023337, 0.011673610228970408, 0.023043760370614948, 0.0095833331794795053, 0.014804248516850377, 0.011115146440537623, 0.010849447538179248, 0.011622884004589837, 0.027129713559273013, 0.012121736636813752, 0.015034681042991226, 0.017562154324935825, 0.014640765599476307, 0.018159047120203501, 0.012057226276762218, 0.022196697102359254, 0.012029792163633852, 0.019705809697590913, 0.0087939803887059995, 0.008912544252171457, 0.01141888979980611] 2018-01-20 01:10:55,922 : INFO : PROGRESS: pass 25, at document #4000/8867 2018-01-20 01:11:00,900 : INFO : optimized alpha [0.033088556643093757, 0.018769397211399828, 0.012266488241344107, 0.014542293394732779, 0.01543179179271828, 0.028372814538111607, 0.01075422393964527, 0.011254195560339573, 0.011349533188141733, 0.01538393008966789, 0.020983151228214307, 0.011313891196953141, 0.016942496192158783, 0.015516054777989063, 0.013637820527573164, 0.018347832460380159, 0.010495199055599383, 0.012581223803435373, 0.017378438881595588, 0.011764108972928003, 0.013572169451734908, 0.020578632883340799, 0.01267080216893432, 0.017611683860629206, 0.013888015564938717, 0.014298641935631311, 0.013450277186588764, 0.018593664893575893, 0.0099602088750463339, 0.018902408510331932, 0.011707271990907819, 0.02316548295397616, 0.0095896239674079821, 0.014814070767943758, 0.011129576294504206, 0.010865292402339397, 0.011644020561774009, 0.027204827946759678, 0.012152789579503127, 0.015075806303785123, 0.017578743102273371, 0.014681200650946065, 0.018204540762472927, 0.012073007955290153, 0.022261701661440626, 0.012076931743240742, 0.019752890004487281, 0.0088004325758811036, 0.0089304822778404173, 0.011437596046848878] 2018-01-20 01:11:00,902 : INFO : PROGRESS: pass 25, at document #6000/8867 2018-01-20 01:11:05,903 : INFO : optimized alpha [0.03323879694054916, 0.018863483541141837, 0.012299585062835001, 0.014571658504978236, 0.015473947726863035, 0.028506843175324377, 0.010760901017297177, 0.011275969820183937, 0.011367534929595594, 0.015444679875264183, 0.020997754972900885, 0.011332537093831115, 0.017004510956752837, 0.015534668076888651, 0.01366613010673046, 0.018429007469737246, 0.010504703433251515, 0.012608409077679789, 0.017463481656855323, 0.011801796211564001, 0.013612295668805723, 0.02059463934863541, 0.01270356856748908, 0.017632197224624252, 0.013926260366763405, 0.014332331451255752, 0.013463535524831479, 0.018665939685056317, 0.009969971431783917, 0.018947322245406989, 0.01171903259614054, 0.023244091119310792, 0.009611190802388302, 0.014828206578742495, 0.011143924148532361, 0.010876517864508353, 0.011659538189462484, 0.027241602427850214, 0.012167783601095697, 0.015090943832349459, 0.017622145472318849, 0.014720455524132226, 0.018264760659955356, 0.012098184263728742, 0.022352946048779725, 0.012098380902938237, 0.019761101983410016, 0.0088086696673601167, 0.0089525814451438916, 0.011447789458467087] 2018-01-20 01:11:05,904 : INFO : PROGRESS: pass 25, at document #8000/8867 2018-01-20 01:11:10,934 : INFO : optimized alpha [0.033307191877743975, 0.018946772852211198, 0.012314864911075736, 0.014596253448724556, 0.015526752241403522, 0.028644009221980921, 0.010774996436804289, 0.011283099024919349, 0.011406784025698238, 0.015476081606452721, 0.02111300056908284, 0.011334348466331339, 0.017064364784990901, 0.015589638970786964, 0.013735718916177116, 0.018462712255648224, 0.010522162260800424, 0.012634520136133854, 0.017525452763516847, 0.011843206042375444, 0.013651410174131989, 0.020704593711951526, 0.012733328372506533, 0.017693027997730242, 0.013950295705657206, 0.014360169345545033, 0.013514837338615224, 0.018714684304537432, 0.0099983018030285319, 0.019030587188909334, 0.011757747393621221, 0.023437528938031676, 0.0096222237002428924, 0.014847275895341875, 0.011151275947500696, 0.010906935369894896, 0.011672995957103973, 0.027340027744358533, 0.012183046490291427, 0.015133110062003657, 0.017632748842903216, 0.014741187544889392, 0.018300144862144652, 0.012093987471820805, 0.022454057435055838, 0.01211547642554273, 0.019821710765768736, 0.0088186839649396845, 0.0089465093238915976, 0.011457919455364049] 2018-01-20 01:11:10,935 : INFO : PROGRESS: pass 25, at document #8867/8867 2018-01-20 01:11:13,134 : INFO : optimized alpha [0.033466127160336354, 0.019061671173175972, 0.01233776402728106, 0.014616167055246078, 0.015535742541090624, 0.028674906758515133, 0.010803790323507825, 0.011266874534095672, 0.011434224753609813, 0.015533303496020936, 0.021193927157246949, 0.011363064958880677, 0.017099821919089583, 0.01561950581769715, 0.013749772877119492, 0.018495459432794136, 0.010527325004594336, 0.012652678727133193, 0.017611475451002098, 0.011847674822920758, 0.013720401434320482, 0.020753010302506363, 0.012752637573894983, 0.017807627418948618, 0.013946424769176404, 0.014398513586773312, 0.013561350137343525, 0.018753178442042738, 0.010009545988283094, 0.019070408203723681, 0.011752876875164785, 0.02352736226444694, 0.0096281241969283782, 0.014881940550991615, 0.011219122938034868, 0.010917584505075795, 0.011693714209435201, 0.027455716628958349, 0.012242533732146966, 0.015202144019509563, 0.017660841209650259, 0.014781428705788991, 0.018364411462939333, 0.012134282241593264, 0.022492998509046402, 0.012145078687414188, 0.019867746367621054, 0.0088230074136308231, 0.0089726079298366717, 0.011468837294889905] 2018-01-20 01:11:13,491 : INFO : topic #47 (0.009): 0.053*"ontology" + 0.017*"workshop" + 0.014*"ceur" + 0.013*"proceeding" + 0.012*"semantic" + 0.009*"language" + 0.009*"semantic_web" + 0.008*"w" + 0.008*"ceur_workshop" + 0.007*"arabic" 2018-01-20 01:11:13,492 : INFO : topic #48 (0.009): 0.032*"configuration" + 0.024*"instruction" + 0.022*"simulation" + 0.015*"simulator" + 0.013*"space" + 0.010*"set" + 0.009*"embedded" + 0.009*"instruction_set" + 0.009*"path" + 0.009*"system" 2018-01-20 01:11:13,494 : INFO : topic #37 (0.027): 0.040*"model" + 0.021*"neural" + 0.014*"data" + 0.013*"learning" + 0.012*"network" + 0.009*"parameter" + 0.008*"bayesian" + 0.007*"gaussian" + 0.007*"using" + 0.007*"neural_network" 2018-01-20 01:11:13,495 : INFO : topic #5 (0.029): 0.024*"system" + 0.021*"web" + 0.017*"service" + 0.009*"application" + 0.009*"technology" + 0.009*"user" + 0.008*"project" + 0.008*"design" + 0.008*"paper" + 0.008*"research" 2018-01-20 01:11:13,496 : INFO : topic #0 (0.033): 0.071*"springer" + 0.057*"berlin" + 0.055*"heidelberg" + 0.032*"berlin_heidelberg" + 0.024*"international" + 0.023*"springer_berlin" + 0.019*"proceeding" + 0.019*"conference" + 0.016*"heidelberg_springer" + 0.011*"verlag" 2018-01-20 01:11:13,499 : INFO : topic diff=0.118660, rho=0.181269 2018-01-20 01:11:13,663 : INFO : PROGRESS: pass 26, at document #2000/8867 2018-01-20 01:11:18,559 : INFO : optimized alpha [0.033490991011377771, 0.019163759295504051, 0.012365708003978501, 0.014622649265863884, 0.015583499828911815, 0.028788408619374106, 0.010831131623021288, 0.011315413649688301, 0.011445985277809408, 0.015553431777428453, 0.021329493682490236, 0.011390340841059695, 0.017119175959496264, 0.015629270539485436, 0.013775158038823917, 0.018570754819455561, 0.010560609279297506, 0.01266258950675558, 0.017655103327017509, 0.011873702021915403, 0.013731700169939805, 0.020739024372765993, 0.012789008136562816, 0.017860676938461273, 0.013995113002192814, 0.014453859073927459, 0.013640239532116414, 0.018830536539556637, 0.010025038394607135, 0.019107020175840102, 0.011750108816749767, 0.023594160887986621, 0.009623448458526862, 0.014923419451102721, 0.011243620742502661, 0.01093098617254987, 0.011733587676801884, 0.027556495503476386, 0.012278914439957553, 0.015263339049416751, 0.017720321280017597, 0.014784766352440162, 0.018419423842208883, 0.012160467462447337, 0.022548309643506792, 0.012183219536344603, 0.019918616518732481, 0.0088486590403953989, 0.0089875118975047651, 0.011502909902522649] 2018-01-20 01:11:18,560 : INFO : PROGRESS: pass 26, at document #4000/8867 2018-01-20 01:11:23,476 : INFO : optimized alpha [0.033512738082344153, 0.019249944160193631, 0.012408213471964021, 0.014651741697282172, 0.015631431621408166, 0.028801217754961209, 0.010837332478767109, 0.011347173396670931, 0.011468162465211351, 0.015604501692686193, 0.021390378469796183, 0.011397742931500175, 0.017160952609114374, 0.015637398102849517, 0.013805748444690381, 0.018573767958478513, 0.01056956426521459, 0.012693392512415677, 0.017708430491242027, 0.011901716053834752, 0.013766236035279667, 0.020766974389036245, 0.012820631711838682, 0.017911318145084777, 0.014044559536773074, 0.014490360622769257, 0.013624806563811053, 0.01888576468561488, 0.010026058804056217, 0.019206604369292161, 0.011785330075527897, 0.023710276863311956, 0.0096296899358939512, 0.014932508298073997, 0.011258211240527805, 0.010943435908302674, 0.011756113240348082, 0.027630199838252125, 0.012313237562575656, 0.015303687633449494, 0.017740437695682891, 0.014826709746052059, 0.018475621710039906, 0.012178893184509878, 0.022606631459389563, 0.01223028093816796, 0.019965212238060914, 0.0088589052038335492, 0.0090074753764482467, 0.011525253905750224] 2018-01-20 01:11:23,477 : INFO : PROGRESS: pass 26, at document #6000/8867 2018-01-20 01:11:28,424 : INFO : optimized alpha [0.033673883078119152, 0.019343803873087047, 0.012443558463124277, 0.014682965337405639, 0.015673210714867686, 0.028922870750527688, 0.010841684332265606, 0.011370434566856671, 0.011487461412678633, 0.015664596250063504, 0.021403434463950679, 0.011418991674143291, 0.017220710242831783, 0.01565222405123189, 0.013832660296380094, 0.018650709214651534, 0.010583469522556654, 0.012719774944499624, 0.017793082598172361, 0.011938371417564042, 0.013808501501194018, 0.020785035729871729, 0.012849932171261772, 0.017934995967696214, 0.014084071196354811, 0.014522352672077974, 0.013634195501296544, 0.018966885244903427, 0.01003734711807119, 0.019248013902320085, 0.011796703901267282, 0.023783188454266625, 0.0096501769427223272, 0.014947223651341476, 0.011270699473007165, 0.010954284452612678, 0.011769842065195541, 0.027663281030420067, 0.01233044601767644, 0.015322202708523698, 0.017779145393558196, 0.014860657212383516, 0.018542967304181226, 0.012202265680682481, 0.022685105621759977, 0.012250627065638552, 0.019969458136087505, 0.0088685070888646477, 0.0090307359531342484, 0.011533962046302923] 2018-01-20 01:11:28,425 : INFO : PROGRESS: pass 26, at document #8000/8867 2018-01-20 01:11:33,423 : INFO : optimized alpha [0.033737607497050494, 0.019432202904051957, 0.012458939247236803, 0.014706801577692554, 0.015726939504062672, 0.029053003485257856, 0.010855370063523023, 0.011377078230671963, 0.011528899540111191, 0.015698256866830587, 0.021505681066603811, 0.011419405126035169, 0.017276823538746523, 0.015701830460153672, 0.013905763233150204, 0.01867513523439435, 0.010603366247655201, 0.01274975523320002, 0.017856295406845697, 0.011985250464866908, 0.013844686696709668, 0.020892584825358267, 0.012880889731806145, 0.017987683923467498, 0.014113201463027726, 0.014549826132328083, 0.013687776383845074, 0.019010347979927098, 0.010067017980667276, 0.019320821896361549, 0.011834806229933584, 0.023984876480908274, 0.009661999094229979, 0.014968202648138529, 0.011277257000927956, 0.010986139346413347, 0.011783668613852209, 0.02775772542242072, 0.012347936428445498, 0.015367656402136935, 0.017794927106059164, 0.014881709769251666, 0.018582089500231557, 0.012196490096288841, 0.022777660932530062, 0.012268867240117463, 0.020031238576919758, 0.0088782669057759934, 0.0090268178146053202, 0.011545722403739172] 2018-01-20 01:11:33,424 : INFO : PROGRESS: pass 26, at document #8867/8867 2018-01-20 01:11:35,593 : INFO : optimized alpha [0.033896164520882219, 0.019566543201716909, 0.012483393743828515, 0.014729044341126617, 0.015734072590199148, 0.029070998635746758, 0.010883635458365723, 0.011365297846519217, 0.011557873237553265, 0.0157500507533593, 0.021590525747887231, 0.011445155883706107, 0.017313853795702116, 0.015733976736819671, 0.013912645090291571, 0.018705744501031622, 0.010614783897993548, 0.012769870889033606, 0.017938778820555005, 0.012001292654691083, 0.013911966413324657, 0.020934450410162148, 0.012907242405666609, 0.018099138131026638, 0.014108080638461577, 0.014586494635501048, 0.013727018385276398, 0.019053591493139961, 0.010080044980587439, 0.019365461931296523, 0.011832226931342893, 0.024071540677858352, 0.0096698648058602434, 0.01500502307357845, 0.011344041334783009, 0.010994104333748209, 0.011801216992079856, 0.027872893654426991, 0.012403522129511039, 0.015434374087171182, 0.017825311580916232, 0.014923815103252788, 0.018647121141941534, 0.012238469599970424, 0.022817376417093013, 0.012297328842967853, 0.020066511075080281, 0.0088824015088763073, 0.0090524359865232686, 0.011556173984964157] 2018-01-20 01:11:35,938 : INFO : topic #47 (0.009): 0.054*"ontology" + 0.018*"workshop" + 0.014*"ceur" + 0.013*"proceeding" + 0.012*"semantic" + 0.010*"semantic_web" + 0.009*"language" + 0.008*"w" + 0.008*"ceur_workshop" + 0.007*"arabic" 2018-01-20 01:11:35,939 : INFO : topic #48 (0.009): 0.033*"configuration" + 0.025*"instruction" + 0.022*"simulation" + 0.016*"simulator" + 0.013*"space" + 0.011*"set" + 0.009*"embedded" + 0.009*"path" + 0.009*"instruction_set" + 0.009*"system" 2018-01-20 01:11:35,941 : INFO : topic #37 (0.028): 0.041*"model" + 0.021*"neural" + 0.014*"data" + 0.013*"learning" + 0.013*"network" + 0.009*"parameter" + 0.008*"bayesian" + 0.007*"gaussian" + 0.007*"using" + 0.007*"neural_network" 2018-01-20 01:11:35,942 : INFO : topic #5 (0.029): 0.024*"system" + 0.021*"web" + 0.017*"service" + 0.009*"application" + 0.009*"technology" + 0.009*"user" + 0.008*"project" + 0.008*"design" + 0.008*"paper" + 0.008*"distributed" 2018-01-20 01:11:35,944 : INFO : topic #0 (0.034): 0.072*"springer" + 0.057*"berlin" + 0.055*"heidelberg" + 0.032*"berlin_heidelberg" + 0.024*"international" + 0.023*"springer_berlin" + 0.019*"proceeding" + 0.019*"conference" + 0.016*"heidelberg_springer" + 0.011*"verlag" 2018-01-20 01:11:35,950 : INFO : topic diff=0.114450, rho=0.178363 2018-01-20 01:11:36,111 : INFO : PROGRESS: pass 27, at document #2000/8867 2018-01-20 01:11:40,986 : INFO : optimized alpha [0.033916819411049653, 0.019668234569418484, 0.012513597131834472, 0.014734908754875211, 0.015785622880452887, 0.029175476397754044, 0.010910476707168725, 0.011414069959473328, 0.011571912934030435, 0.015771008002175433, 0.021723264766588229, 0.011469864308563166, 0.017326479348799211, 0.015743075712921422, 0.01393549573571386, 0.018787189532956258, 0.010646531673290055, 0.012782507087526861, 0.017981997790250861, 0.012029463485177817, 0.013922874151679839, 0.02093048382235466, 0.012937836015419849, 0.018142468109960192, 0.014155387732955908, 0.014640208692695571, 0.013804420197885496, 0.019129362485422796, 0.010095160135550868, 0.019404000973953683, 0.011829236460554051, 0.024132442278506607, 0.0096652586842543558, 0.01504553503518829, 0.011368672640165743, 0.011007121545905085, 0.011843425401609205, 0.027974975005303299, 0.012445398294565724, 0.015497605221958272, 0.017886427004482584, 0.014930187980890023, 0.018705786439493704, 0.012263943354709718, 0.02286825122716335, 0.012335371669315103, 0.020115234903361107, 0.0089045443526729971, 0.0090677131984545634, 0.011586547910054222] 2018-01-20 01:11:40,988 : INFO : PROGRESS: pass 27, at document #4000/8867 2018-01-20 01:11:45,907 : INFO : optimized alpha [0.033934341398169657, 0.019759005796030297, 0.012553828239982859, 0.01476713510754972, 0.015838726852201584, 0.029176842794478047, 0.010916369471160177, 0.01144720598901394, 0.011596312451672749, 0.015818636315333468, 0.02178108050081453, 0.011480989787861709, 0.017363115324278561, 0.015749136467390634, 0.013964801279685798, 0.018791501290448442, 0.010655084384085705, 0.012810166036835085, 0.018031876017205652, 0.012059572016987459, 0.013955758201120352, 0.020962251387923528, 0.012972961255914144, 0.018196449602005508, 0.014207238439485171, 0.014676390162793649, 0.01378908523245089, 0.019182864927705625, 0.010100432731390673, 0.019501972091916835, 0.011868077191921721, 0.02424766065846715, 0.0096688902205726256, 0.015056575706202448, 0.011387474963227638, 0.011022138283490319, 0.011866253266669066, 0.028050074710159009, 0.012478463313844296, 0.015535722018134513, 0.017901846184059614, 0.014970925692249511, 0.018763921463041382, 0.01228282654890255, 0.022922580455521306, 0.012383435853553457, 0.020152579043888747, 0.008912226252183084, 0.009087239048863726, 0.011607326696583865] 2018-01-20 01:11:45,909 : INFO : PROGRESS: pass 27, at document #6000/8867 2018-01-20 01:11:50,870 : INFO : optimized alpha [0.034081883139715899, 0.019846951140337878, 0.012588030909868438, 0.014797443756458903, 0.015874305760844445, 0.029319360421751518, 0.010921372354870248, 0.011468819915641261, 0.011614768791350623, 0.015878128539781853, 0.021796709636431016, 0.011502764386510355, 0.017417763822743355, 0.015761862907480998, 0.013992978410384013, 0.018867817432278916, 0.010670456115295308, 0.012840309882023122, 0.018121316384029258, 0.012096141722326094, 0.013995236437027681, 0.020976591389995667, 0.013003432212127628, 0.018216962094048321, 0.014247843056918461, 0.014710705219881024, 0.013802105003922325, 0.019255581225152914, 0.010111364470168784, 0.019539949683285093, 0.011880074822219601, 0.024316923605282054, 0.0096883172581572995, 0.015067793695877107, 0.01139910732543762, 0.011036534733677316, 0.01188038655944338, 0.028094616665319407, 0.01249229496419095, 0.015550652790316484, 0.017942330951407227, 0.015010430430500512, 0.018828218973734469, 0.012310936450340541, 0.022996976520962133, 0.012402647230268463, 0.020157084467431405, 0.0089224275753781029, 0.0091084274851243718, 0.011615654515610346] 2018-01-20 01:11:50,871 : INFO : PROGRESS: pass 27, at document #8000/8867 2018-01-20 01:11:55,872 : INFO : optimized alpha [0.034138602059451587, 0.019941974039763482, 0.01260128843805422, 0.014824444677399472, 0.015927636253494016, 0.029432631070403097, 0.010934664765878149, 0.011478033764300248, 0.011657360113033848, 0.015908431037032423, 0.021895942745548035, 0.011502833526344496, 0.017475097504076147, 0.015813302393615305, 0.014067121422649464, 0.01889351538249795, 0.010693676934371428, 0.012868270472918619, 0.018184308633227547, 0.012139705847353376, 0.014033643250317092, 0.021083821785159162, 0.01303205580581099, 0.018276426805745202, 0.014275709172270817, 0.014737864549339265, 0.0138505304595519, 0.019302789568434505, 0.010139652636933972, 0.019620062698988448, 0.011917538562457428, 0.02452074151461521, 0.0097017304674291097, 0.015086603592419144, 0.011408881106884991, 0.011064858885310768, 0.011891428260187097, 0.028187592076816731, 0.012517495453736573, 0.01559392333401668, 0.017963197028715312, 0.015030294727510088, 0.018860723437478293, 0.012310086049649479, 0.023093899786302927, 0.012420855623806511, 0.020211546533351905, 0.008932782222288483, 0.0091042479372139256, 0.011627097646841389] 2018-01-20 01:11:55,874 : INFO : PROGRESS: pass 27, at document #8867/8867 2018-01-20 01:11:58,103 : INFO : optimized alpha [0.034290174110616412, 0.020066479393863627, 0.012629763740734009, 0.014842798460129945, 0.015913156940787602, 0.029438637916880658, 0.010957982555669318, 0.011465833139881339, 0.01168303430801268, 0.015954910288806634, 0.021975547844830093, 0.011532732127640843, 0.017506536217197357, 0.015844610648791089, 0.014081427479612441, 0.018917988381446826, 0.010708980232592582, 0.01289014653841648, 0.018259489378295447, 0.01214724091471211, 0.014099292333629552, 0.021123959650958972, 0.013057230272901272, 0.018380978743228582, 0.014263567239300685, 0.014778890510856828, 0.013885517047126841, 0.019338553853674837, 0.010150226372916798, 0.019657103983637795, 0.011917083365006131, 0.024609630859974932, 0.0097095229560106022, 0.01511934604810322, 0.011474588718292733, 0.011074734086569232, 0.011910727941885539, 0.028308040739245602, 0.012582046167450314, 0.015661673006575744, 0.017999483631250295, 0.015077346021057919, 0.018934477305717564, 0.012343592173463955, 0.023134223507353047, 0.012448175365847913, 0.020266106763287214, 0.0089385729220992065, 0.0091312437850551622, 0.011634704636253945] 2018-01-20 01:11:58,442 : INFO : topic #47 (0.009): 0.056*"ontology" + 0.018*"workshop" + 0.014*"ceur" + 0.013*"proceeding" + 0.013*"semantic" + 0.010*"semantic_web" + 0.009*"language" + 0.008*"w" + 0.008*"ceur_workshop" + 0.006*"measurement" 2018-01-20 01:11:58,444 : INFO : topic #48 (0.009): 0.033*"configuration" + 0.026*"instruction" + 0.023*"simulation" + 0.016*"simulator" + 0.014*"space" + 0.011*"set" + 0.009*"path" + 0.009*"embedded" + 0.009*"instruction_set" + 0.009*"system" 2018-01-20 01:11:58,445 : INFO : topic #37 (0.028): 0.041*"model" + 0.021*"neural" + 0.014*"data" + 0.014*"learning" + 0.013*"network" + 0.009*"parameter" + 0.008*"bayesian" + 0.007*"gaussian" + 0.007*"using" + 0.007*"neural_network" 2018-01-20 01:11:58,446 : INFO : topic #5 (0.029): 0.024*"system" + 0.021*"web" + 0.017*"service" + 0.009*"application" + 0.009*"technology" + 0.008*"user" + 0.008*"project" + 0.008*"design" + 0.008*"paper" + 0.008*"support" 2018-01-20 01:11:58,447 : INFO : topic #0 (0.034): 0.072*"springer" + 0.058*"berlin" + 0.055*"heidelberg" + 0.032*"berlin_heidelberg" + 0.025*"international" + 0.023*"springer_berlin" + 0.019*"proceeding" + 0.019*"conference" + 0.016*"heidelberg_springer" + 0.012*"verlag" 2018-01-20 01:11:58,450 : INFO : topic diff=0.111186, rho=0.175591 2018-01-20 01:11:58,618 : INFO : PROGRESS: pass 28, at document #2000/8867 2018-01-20 01:12:03,567 : INFO : optimized alpha [0.034316656599356045, 0.020167916349037605, 0.012660034231759772, 0.01484537175854469, 0.015963058165512602, 0.02955360819326892, 0.010985329903764969, 0.011515838340587423, 0.011699326170856806, 0.015976862005099509, 0.022101514503734303, 0.011558922690704142, 0.017523645411332629, 0.015853046116657728, 0.014110341625104101, 0.018995422911383907, 0.010741998102067435, 0.012907698065300266, 0.01830574388929062, 0.012173317009711056, 0.014111240469494881, 0.021116448303239069, 0.013087850017511756, 0.018424508477697873, 0.014312090612202595, 0.014830873077305719, 0.013960274613640793, 0.019412800542783368, 0.010165002651774196, 0.019683581614135467, 0.011916886210640682, 0.024662487643768098, 0.009707507223334421, 0.015158887222318935, 0.011497354410195927, 0.011088347665696292, 0.01194998978474422, 0.028406126057817487, 0.012624829790188441, 0.015721354213292005, 0.018054093742413891, 0.01508392466305321, 0.018988271094014506, 0.012368401061157851, 0.023181232683715542, 0.012487266186277584, 0.020325312879115504, 0.0089611681392162811, 0.0091501064342557111, 0.01166966724832719] 2018-01-20 01:12:03,569 : INFO : PROGRESS: pass 28, at document #4000/8867 2018-01-20 01:12:08,485 : INFO : optimized alpha [0.034330432045271524, 0.020251082844947643, 0.012701313465609376, 0.014880714101293418, 0.016010336777219022, 0.029546877461637751, 0.010992854337881946, 0.011549269898001865, 0.011722873755230347, 0.016023973068061467, 0.022152369595654416, 0.011568570816574475, 0.017563014036460926, 0.015857095736035148, 0.014141914441612651, 0.019001239449758477, 0.010755701340424528, 0.012938958361343254, 0.018355260492812442, 0.012205549030066971, 0.014142574486873939, 0.021140763240325901, 0.013120679161761718, 0.018473743603889824, 0.014360096747452021, 0.01486674534806382, 0.013948783074268393, 0.019464860270233423, 0.010169999371791184, 0.019783611709080422, 0.011956098472410994, 0.024776866801510353, 0.009712777211217348, 0.015169276627198454, 0.011514293828867378, 0.011104853521207067, 0.011974234094976163, 0.028471917252239121, 0.012654331976170543, 0.01575597140592928, 0.018066497139054494, 0.015123262535955476, 0.019038225915607526, 0.012386609687176996, 0.023235540043858999, 0.012535254679497738, 0.02036215536267225, 0.0089686476289261609, 0.0091715838278759298, 0.011693017997663582] 2018-01-20 01:12:08,486 : INFO : PROGRESS: pass 28, at document #6000/8867 2018-01-20 01:12:13,510 : INFO : optimized alpha [0.034480423545959232, 0.020340417224637308, 0.012735512505290592, 0.014911466106957561, 0.016052823759179467, 0.02968132691993133, 0.010999491530620166, 0.011571250372317123, 0.011743571771514237, 0.01608580521313124, 0.022165166174903914, 0.011587786241956539, 0.017615773319400636, 0.015871976233073508, 0.014169878601925449, 0.019077023161431329, 0.010768580812808863, 0.012967040844857853, 0.018441236117771667, 0.012241072104315159, 0.014185488287879522, 0.021151617766223135, 0.013154641992656469, 0.018489646095393331, 0.014399443152927733, 0.014899457840053979, 0.013964325530096139, 0.019532762488714734, 0.010183312458657903, 0.019821974707614837, 0.011962413084811864, 0.024842680397025248, 0.0097328358517963665, 0.015182505112652619, 0.011525096247929413, 0.01111780410045763, 0.011987707420666809, 0.028509882345596413, 0.012670175999612813, 0.015771620752737039, 0.018105470102057212, 0.015160141867895122, 0.019107939508968188, 0.012410767383782564, 0.023306197713389872, 0.012557742800401549, 0.020371799568140149, 0.0089794032517669549, 0.0091947090807997598, 0.011700926820167634] 2018-01-20 01:12:13,512 : INFO : PROGRESS: pass 28, at document #8000/8867 2018-01-20 01:12:18,532 : INFO : optimized alpha [0.03453042811393333, 0.020438820996419773, 0.012748832409741152, 0.014938991020393892, 0.016103047136527854, 0.029791485880888931, 0.011011454679942778, 0.011582945826952695, 0.011787273858605897, 0.016114221968231531, 0.022263691702663471, 0.011587480968621836, 0.017674205778693188, 0.015916852243382979, 0.014240030173198852, 0.019103992977457702, 0.010794033964695405, 0.012994053120449371, 0.018502410318611433, 0.012288949779803578, 0.014222310708135731, 0.021258656690020944, 0.013185605336350735, 0.018549444660423551, 0.014426116614369468, 0.014926332388258528, 0.01401261865165316, 0.019575141580672382, 0.010211102618782619, 0.019902400342870601, 0.01200557714366105, 0.025048721408748278, 0.009746939569927544, 0.015203306780828553, 0.011532069205985214, 0.011147541956640547, 0.012001268076996871, 0.028596453092404436, 0.012689644867799372, 0.015816965687262024, 0.018126548875366078, 0.01518019813813081, 0.019144239907054588, 0.012410541479120156, 0.023397033306619162, 0.012578178250069892, 0.020429333499111763, 0.0089903109368728099, 0.0091925812984555103, 0.011712952058596274] 2018-01-20 01:12:18,533 : INFO : PROGRESS: pass 28, at document #8867/8867 2018-01-20 01:12:20,769 : INFO : optimized alpha [0.034689012810551346, 0.020566157748561679, 0.012776193150053641, 0.014953580302051241, 0.016090493066809895, 0.029785996582108711, 0.011034317763511766, 0.011574983192999017, 0.011816833360248151, 0.016162032332933677, 0.022352348166968135, 0.011612095132517705, 0.017703730959435601, 0.015956988053260391, 0.014255831018982061, 0.019110529967876509, 0.010804400346997733, 0.01301502319182949, 0.018574092896449831, 0.012300357283594852, 0.014289138471766201, 0.021297048233774544, 0.013209675496791891, 0.018647221398819094, 0.014421651356269157, 0.014968733939744955, 0.014040566771367802, 0.019607991659466324, 0.010219285099171424, 0.019940211067674223, 0.012004728600803956, 0.025135130688371143, 0.0097546069603030722, 0.015238323784439867, 0.011599094244112552, 0.011157005082706718, 0.012024711322388686, 0.028716782654111664, 0.012758020128265395, 0.015892166157653195, 0.018150362084255363, 0.015219823545144645, 0.01921468258865739, 0.012448346366367463, 0.023433720857970089, 0.012606908631831721, 0.02047737436055903, 0.0089958970704058974, 0.0092208912006541618, 0.011722562548494037] 2018-01-20 01:12:21,100 : INFO : topic #47 (0.009): 0.056*"ontology" + 0.018*"workshop" + 0.014*"semantic" + 0.014*"ceur" + 0.013*"proceeding" + 0.010*"semantic_web" + 0.009*"language" + 0.008*"w" + 0.008*"ceur_workshop" + 0.007*"knowledge" 2018-01-20 01:12:21,101 : INFO : topic #48 (0.009): 0.033*"configuration" + 0.026*"instruction" + 0.024*"simulation" + 0.016*"simulator" + 0.014*"space" + 0.011*"set" + 0.010*"path" + 0.009*"embedded" + 0.009*"instruction_set" + 0.009*"system" 2018-01-20 01:12:21,103 : INFO : topic #37 (0.029): 0.041*"model" + 0.021*"neural" + 0.014*"data" + 0.014*"learning" + 0.013*"network" + 0.009*"parameter" + 0.008*"bayesian" + 0.007*"gaussian" + 0.007*"using" + 0.007*"neural_network" 2018-01-20 01:12:21,104 : INFO : topic #5 (0.030): 0.024*"system" + 0.021*"web" + 0.017*"service" + 0.009*"application" + 0.009*"technology" + 0.008*"user" + 0.008*"project" + 0.008*"design" + 0.008*"support" + 0.008*"paper" 2018-01-20 01:12:21,105 : INFO : topic #0 (0.035): 0.073*"springer" + 0.058*"berlin" + 0.056*"heidelberg" + 0.032*"berlin_heidelberg" + 0.025*"international" + 0.023*"springer_berlin" + 0.019*"proceeding" + 0.019*"conference" + 0.017*"heidelberg_springer" + 0.012*"verlag" 2018-01-20 01:12:21,108 : INFO : topic diff=0.108625, rho=0.172945 2018-01-20 01:12:21,285 : INFO : PROGRESS: pass 29, at document #2000/8867 2018-01-20 01:12:26,241 : INFO : optimized alpha [0.03471195207180721, 0.020670736301087587, 0.012804293045872614, 0.014960710732264524, 0.016143053291637406, 0.029887784378006349, 0.011060237020245922, 0.011625134668999781, 0.011832251209141798, 0.016180799494458623, 0.022475599214934704, 0.011634775344818478, 0.017719031637855799, 0.015964732080152425, 0.014282100478397057, 0.01919092731259741, 0.010837684462337219, 0.013029496795247533, 0.018618604856793074, 0.012326421602405869, 0.014298305209572483, 0.021288133586340831, 0.013244821434311377, 0.018687888971034908, 0.014471294336367015, 0.015017744887299738, 0.014111545313976131, 0.019681053221745473, 0.0102328349142413, 0.019970451353449679, 0.012006245053755943, 0.025184568380020988, 0.0097508867176834563, 0.015278281533265265, 0.011623031835416032, 0.011173086816473141, 0.012067311794657174, 0.028818829508081675, 0.012800605099872563, 0.015952442984874412, 0.018203540882999691, 0.015225424799964919, 0.019272033987225032, 0.012476732721435485, 0.023468665295959229, 0.012644810874055501, 0.020530741444565282, 0.0090197218943182042, 0.0092408352522115669, 0.011754871882129809] 2018-01-20 01:12:26,242 : INFO : PROGRESS: pass 29, at document #4000/8867 2018-01-20 01:12:31,173 : INFO : optimized alpha [0.034731109656100878, 0.020762451053934258, 0.012845629342913313, 0.014992626050636674, 0.016195729369869807, 0.029884297290891836, 0.01106647455535961, 0.011654784143605318, 0.011856939577044026, 0.016228882080339197, 0.022527578364189275, 0.01164408744593478, 0.017756511605934139, 0.015969551982787565, 0.014308594826630897, 0.019198252194696105, 0.010849927507749088, 0.013062140656922499, 0.01866158782611688, 0.012354384818807211, 0.014330630006823537, 0.021314682075609052, 0.013281025220306034, 0.018737568825844837, 0.014517977370751721, 0.015050668187618039, 0.014095369974844887, 0.019731985070691377, 0.010235852266233091, 0.020063723911457693, 0.012046878172747839, 0.025291438157140338, 0.0097560789198038438, 0.015287922826630055, 0.0116411365735784, 0.011190097742316444, 0.012091790211904346, 0.028880918462196547, 0.012835479253675289, 0.015987691480478381, 0.018216385891360674, 0.015268913870444052, 0.019318854223712961, 0.012496442327640484, 0.023521554910895574, 0.012689449599827393, 0.020569093789411703, 0.0090277505853797683, 0.0092626077912844294, 0.011778762843145076] 2018-01-20 01:12:31,174 : INFO : PROGRESS: pass 29, at document #6000/8867 2018-01-20 01:12:36,004 : INFO : optimized alpha [0.034877339999440295, 0.020851056261728881, 0.012880950919339541, 0.015022597848528885, 0.0162352515507714, 0.030021462277041715, 0.011073785016095907, 0.011677110228510869, 0.01187468398972753, 0.016288838023925049, 0.022537299872243816, 0.011661863306409748, 0.017804289731217292, 0.015983706538412246, 0.014336441346829992, 0.019275307178923905, 0.010862295020264647, 0.013089403834895106, 0.018749318639161187, 0.012389895180109795, 0.014371973385134533, 0.021327815921279601, 0.013312637719787196, 0.018753974513655906, 0.014552296909507254, 0.015081738904367675, 0.014111011394300061, 0.019796980929197475, 0.010250609440947583, 0.020102364206620787, 0.012055854581865106, 0.025354229936833766, 0.0097759382924348243, 0.015301745310745431, 0.011654134594770714, 0.011204523395083316, 0.012105565467646975, 0.028925727382793472, 0.012854458290371882, 0.016002529467365832, 0.018255634412718623, 0.015309883750698531, 0.019395592327167928, 0.012520954667570545, 0.023586558459567857, 0.012711947075366731, 0.020571263366092971, 0.0090382802713736658, 0.0092844055471593082, 0.011790317127604281] 2018-01-20 01:12:36,005 : INFO : PROGRESS: pass 29, at document #8000/8867 2018-01-20 01:12:40,878 : INFO : optimized alpha [0.034923790905505438, 0.020945211462522029, 0.012889911927314878, 0.015046658422933398, 0.016285207294110298, 0.030137392806223351, 0.011084432272361647, 0.011691227666283307, 0.011917430979702191, 0.016313998774646853, 0.022636752031929677, 0.011662313501624567, 0.017857792712571704, 0.016031717030820555, 0.014410115468240682, 0.019305349537397737, 0.010886200055712556, 0.013122418913593625, 0.018807123235298522, 0.012436694232077312, 0.014409741911032205, 0.021432618640147529, 0.013344674646834507, 0.018812549624098202, 0.014580288901546144, 0.015108412923120755, 0.014160634348051848, 0.019831171678430033, 0.010278853691353422, 0.020176071294752386, 0.012103506022132131, 0.025549299163588789, 0.0097890395882296744, 0.015323057621649349, 0.011662290859558222, 0.011233674673654648, 0.012117345408462356, 0.029011093638868762, 0.01287254714165883, 0.016044201090791663, 0.018273905278433006, 0.015331439331346945, 0.0194321011993395, 0.012522225221481397, 0.02367400419624744, 0.012731287909924286, 0.02062304240067846, 0.0090497250050152397, 0.0092818973641974039, 0.011803917833099625] 2018-01-20 01:12:40,879 : INFO : PROGRESS: pass 29, at document #8867/8867 2018-01-20 01:12:42,523 : INFO : optimized alpha [0.035075571912915854, 0.021062553972985194, 0.012916282410525604, 0.015063605446230028, 0.016277736199558756, 0.030145446498382293, 0.011113482547885523, 0.011680393269999274, 0.011945997277937182, 0.016363305000262538, 0.022706983679168534, 0.011686459223140816, 0.017885579300339335, 0.01606774649146905, 0.014418804908640727, 0.019306027102270779, 0.010898259534456443, 0.013139890037903411, 0.018871778154994061, 0.012444626790657649, 0.014475006714980422, 0.02148669598770302, 0.013375543321275765, 0.01891125458124485, 0.014574733932206405, 0.015146201537393838, 0.014190293778155732, 0.019865341365274531, 0.010290841270369579, 0.020210791170532323, 0.012099765386342824, 0.0256437331610932, 0.0097985772054557298, 0.015357169397455978, 0.011732910766303393, 0.011240414054991998, 0.012137616170061807, 0.029137044419289791, 0.012936953922190129, 0.016114173422991328, 0.018303789318889954, 0.015369818637810657, 0.01950729036535544, 0.012556714680976815, 0.02370723429542803, 0.01276142064647773, 0.020669092453019889, 0.0090515365536044407, 0.0093078366907903189, 0.011817757433895367] 2018-01-20 01:12:42,825 : INFO : topic #47 (0.009): 0.057*"ontology" + 0.018*"workshop" + 0.014*"semantic" + 0.014*"ceur" + 0.013*"proceeding" + 0.010*"semantic_web" + 0.009*"language" + 0.008*"w" + 0.008*"ceur_workshop" + 0.007*"knowledge" 2018-01-20 01:12:42,827 : INFO : topic #48 (0.009): 0.033*"configuration" + 0.027*"instruction" + 0.024*"simulation" + 0.016*"simulator" + 0.014*"space" + 0.011*"set" + 0.010*"path" + 0.010*"embedded" + 0.009*"instruction_set" + 0.009*"system" 2018-01-20 01:12:42,827 : INFO : topic #37 (0.029): 0.041*"model" + 0.021*"neural" + 0.014*"data" + 0.014*"learning" + 0.013*"network" + 0.009*"parameter" + 0.008*"bayesian" + 0.007*"gaussian" + 0.007*"using" + 0.007*"neural_network" 2018-01-20 01:12:42,828 : INFO : topic #5 (0.030): 0.024*"system" + 0.021*"web" + 0.017*"service" + 0.009*"application" + 0.009*"technology" + 0.008*"project" + 0.008*"user" + 0.008*"support" + 0.008*"distributed" + 0.008*"paper" 2018-01-20 01:12:42,829 : INFO : topic #0 (0.035): 0.073*"springer" + 0.058*"berlin" + 0.056*"heidelberg" + 0.032*"berlin_heidelberg" + 0.025*"international" + 0.023*"springer_berlin" + 0.020*"proceeding" + 0.019*"conference" + 0.017*"heidelberg_springer" + 0.012*"verlag" 2018-01-20 01:12:42,831 : INFO : topic diff=0.106639, rho=0.170416
In [32]:
# Save the model:
lda.save('../data/fullpub/fullpub.ldamodel')
2018-01-20 01:20:17,060 : INFO : saving LdaState object under ../data/fullpub/fullpub.ldamodel.state, separately None 2018-01-20 01:20:17,084 : INFO : saved ../data/fullpub/fullpub.ldamodel.state 2018-01-20 01:20:17,089 : INFO : saving LdaModel object under ../data/fullpub/fullpub.ldamodel, separately ['expElogbeta', 'sstats'] 2018-01-20 01:20:17,090 : INFO : storing np array 'expElogbeta' to ../data/fullpub/fullpub.ldamodel.expElogbeta.npy 2018-01-20 01:20:17,097 : INFO : not storing attribute id2word 2018-01-20 01:20:17,098 : INFO : not storing attribute dispatcher 2018-01-20 01:20:17,099 : INFO : not storing attribute state 2018-01-20 01:20:17,101 : INFO : saved ../data/fullpub/fullpub.ldamodel
Evaluate the model¶
In [33]:
top_topics = lda.top_topics(corpus, topn=10)
# Average topic coherence is the sum of topic coherences of all topics, divided by the number of topics.
avg_topic_coherence = sum([t[1] for t in top_topics]) / num_topics
print('Average topic coherence: %.4f.' % avg_topic_coherence)
from pprint import pprint
pprint(sorted(top_topics, key=lambda x:x[1]))
2018-01-20 01:20:20,131 : INFO : CorpusAccumulator accumulated stats from 1000 documents 2018-01-20 01:20:20,156 : INFO : CorpusAccumulator accumulated stats from 2000 documents 2018-01-20 01:20:20,184 : INFO : CorpusAccumulator accumulated stats from 3000 documents 2018-01-20 01:20:20,210 : INFO : CorpusAccumulator accumulated stats from 4000 documents 2018-01-20 01:20:20,238 : INFO : CorpusAccumulator accumulated stats from 5000 documents 2018-01-20 01:20:20,266 : INFO : CorpusAccumulator accumulated stats from 6000 documents 2018-01-20 01:20:20,295 : INFO : CorpusAccumulator accumulated stats from 7000 documents 2018-01-20 01:20:20,328 : INFO : CorpusAccumulator accumulated stats from 8000 documents
Average topic coherence: -2.7474. [([(0.027666118431970629, 'et'), (0.027510583309995725, 'al'), (0.026319958449549378, 'et_al'), (0.022480057987220539, 'stimulus'), (0.0162645609675705, 'frequency'), (0.013959520593875676, 'voting'), (0.010712401281760407, 'speed'), (0.0099045388451562792, 'election'), (0.0097372354811260265, 'value'), (0.0083783445088398967, 'activity')], -8.6599100025383624), ([(0.016303420931052033, 'code'), (0.013782355099412164, 'de'), (0.013454365395046873, 'source'), (0.013346327491286853, 'gait'), (0.011434320323465278, 'walking'), (0.010323310685669797, 'description'), (0.010190209418495825, 'control'), (0.0097634517422045142, 'foot'), (0.0096386160866209734, 'model'), (0.0076556115231147378, 'source_code')], -7.2321036062008908), ([(0.020311849518635967, 'group'), (0.019922081135563756, 'patient'), (0.011470114071131805, 'function'), (0.010903451796898155, 'degree'), (0.0086298495667414075, 'threshold'), (0.0074328809686975668, 'result'), (0.0074214518832041174, 'coral'), (0.0065803373573228331, 'line'), (0.0064004002857505869, 'threshold_function'), (0.0062005704779136877, 'linear')], -6.145977321005164), ([(0.044189528762954247, 'game'), (0.0097375073645935319, 'player'), (0.0093454841154876206, 'brain'), (0.0086576813027720821, 'method'), (0.0083572094966689655, 'trajectory'), (0.0082981336700264745, 'activity'), (0.0080474825210995055, 'using'), (0.0077535231022484814, 'matter'), (0.00745828315909371, 'analysis'), (0.0074347646425322999, 'fish')], -5.5331611966776917), ([(0.012588729082605103, 'model'), (0.0097669334982206386, 'representation'), (0.009470129114825452, 'information'), (0.0093528164614633622, 'system'), (0.008129687763890385, 'drosophila'), (0.0074340631768895727, 'brain'), (0.0066845436024239018, 'body'), (0.0065920288666404996, 'insect'), (0.0065098650738397833, 'text'), (0.0061121019670560897, 'entity')], -5.497084484667484), ([(0.032176017231515974, 'user'), (0.028736944478169869, 'search'), (0.016295752790899298, 'policy'), (0.014650978260077947, 'xml'), (0.010789051948554717, 'compression'), (0.010759438757843454, 'system'), (0.0094255880950705317, 'document'), (0.0086282732405951919, 'interface'), (0.0082585136555799587, 'access'), (0.0082486717364503737, 'xpath')], -4.7317959860043004), ([(0.018217227417170089, 'transformation'), (0.015910891755826953, 'flow'), (0.015884375562827804, 'network'), (0.010168517853294787, 'set'), (0.010103945244549935, 'channel'), (0.0094715274089554411, 'control'), (0.009365625640749841, 'approach'), (0.0088206911885507187, 'physic'), (0.0075810702225451148, 'history'), (0.006678201900647455, 'bidirectional')], -4.578177329967585), ([(0.018422677319974004, 'logic'), (0.013772269958315915, 'tracing'), (0.013196350595713087, 'scheme'), (0.012720487882342459, 'system'), (0.012604084819416449, 'calculus'), (0.010814847554029283, 'traitor'), (0.010354244537986227, 'key'), (0.0087475036191029358, 'nominal'), (0.008480263857942821, 'based'), (0.0075242616442369866, 'model')], -4.484472671671532), ([(0.063861234585517582, 'data'), (0.044844820005066906, 'xml'), (0.021700428944869302, 'database'), (0.020715660834362166, 'schema'), (0.014965090108988933, 'relational'), (0.01392320286237224, 'document'), (0.0098085926933919008, 'exchange'), (0.0092837245022971259, 'constraint'), (0.0083352662606715616, 'source'), (0.0076882033639698467, 'uml')], -4.0591396381899933), ([(0.053268020026380432, 'data'), (0.015429091425555255, 'challenge'), (0.011432610761650778, 'knowledge'), (0.009828452180078202, 'target'), (0.0090212005125342581, 'system'), (0.0089570129271554725, 'intensive'), (0.007436659751804027, 'different'), (0.0073090953227450389, 'unit'), (0.0072392706478277453, 'data_intensive'), (0.0071900787910098461, 'source')], -3.2802788368655422), ([(0.086424706839943077, 'tree'), (0.022186918345816469, 'transducer'), (0.017812959209489519, 'evolution'), (0.013116196277298674, 'automaton'), (0.01026885252731808, 'output'), (0.0093078550473993209, 'tree_transducer'), (0.008999205248259777, 'context'), (0.0089740117145397699, 'grammar'), (0.0079536856948789342, 'algorithm'), (0.0074562389523906867, 'input')], -3.0450262060513942), ([(0.025410454234328288, 'retrieval'), (0.023202295199138353, 'task'), (0.01810959698814214, 'evaluation'), (0.014154316736174033, 'search'), (0.014111507957531572, 'result'), (0.013445668612459475, 'child'), (0.011692386799187193, 'metric'), (0.010659054834912647, 'information'), (0.0088256828851126252, 'information_retrieval'), (0.0077739537926421751, 'document')], -2.9928457305050893), ([(0.032238095711583946, 'software'), (0.019712752763206179, 'engineering'), (0.013922355653060017, 'ieee'), (0.011874997023585698, 'software_engineering'), (0.011358405780238419, 'system'), (0.010765662437319043, 'grained'), (0.01072429134069533, 'computer_science'), (0.010261139007738567, 'science'), (0.010227562824880002, 'proceeding'), (0.00939084356770435, 'service')], -2.8182378164929478), ([(0.01909491178426068, 'network'), (0.012404569718240081, 'measure'), (0.0099991482767460595, 'data'), (0.0094646860462906674, 'spike'), (0.0090676866292766448, 'protein'), (0.0089833539770438255, 'information'), (0.008277436229507433, 'based'), (0.0071960034594927618, 'dependence'), (0.0068135314925823094, 'signal'), (0.0065898613754099668, 'method')], -2.7892531084576699), ([(0.022000457939483905, 'code'), (0.014977960407620823, 'optimization'), (0.011517820826318371, 'compiler'), (0.01044486374425675, 'cue'), (0.0081350212223681301, 'program'), (0.0081142154545403421, 'scene'), (0.0074600317071115262, 'performance'), (0.0069736722613379203, 'dynamic'), (0.0063214778094285232, 'acm'), (0.0060787919444667046, 'technique')], -2.7398706368864447), ([(0.021809320918646284, 'user'), (0.018353743402359683, 'topic'), (0.011134323004238347, 'computer'), (0.0091353586274441033, 'information'), (0.0085188860626592769, 'system'), (0.0083458834744158071, 'tweet'), (0.0079976283986256929, 'news'), (0.0063903914080231808, 'new'), (0.0061317510632894695, 'time'), (0.0060883107733832728, 'story')], -2.6959037465439906), ([(0.035005620184006117, 'dialogue'), (0.023066795501543058, 'user'), (0.016128383829981014, 'task'), (0.014097925780574179, 'system'), (0.010046979113476491, 'feature'), (0.0094399733086868998, 'spoken'), (0.009351782040305295, 'study'), (0.0092164023272484838, 'interaction'), (0.009163131287324074, 'older'), (0.0090665347673272428, 'strategy')], -2.4360083981937093), ([(0.03603974868557195, 'network'), (0.014227435419296676, 'performance'), (0.014028021016281395, 'wireless'), (0.010720392063736711, 'protocol'), (0.010287994688103354, 'sensor'), (0.010105902842321067, 'communication'), (0.010002799645677322, 'ad'), (0.0093777602139481428, 'mobile'), (0.0089856731435439523, 'hoc'), (0.0084111849883261817, 'routing')], -2.4107329016470773), ([(0.036549607260779785, 'artificial'), (0.036013770877327801, 'intelligence'), (0.025170422312740089, 'artificial_intelligence'), (0.022936535282916243, 'distribution'), (0.017822684652017747, 'probability'), (0.015566665755085489, 'conference'), (0.015352788733641671, 'proceeding'), (0.013107150350063972, 'monte'), (0.01297375689725771, 'carlo'), (0.012973110353960638, 'monte_carlo')], -2.339694172712198), ([(0.02603531561570234, 'quantum'), (0.02224853408707661, 'protocol'), (0.013960664747632504, 'security'), (0.012768364431734623, 'computation'), (0.012630639883366705, 'theory'), (0.011939773626828083, 'relation'), (0.009428509174764203, 'structure'), (0.0092734666234700266, 'one'), (0.0089254699176847981, 'category'), (0.0070170148067619431, 'notion')], -2.332332858004655), ([(0.029147361792041783, 'word'), (0.024408487822546177, 'effect'), (0.018915221965349973, 'model'), (0.016775102251298021, 'cognitive'), (0.014697055792305411, 'processing'), (0.012978812817188564, 'visual'), (0.010383839946303462, 'reading'), (0.0095565704883239719, 'science'), (0.0095484540763544155, 'language'), (0.0095157200487952596, 'eye')], -2.3131415466577265), ([(0.054001231537778235, 'data'), (0.024846001892734906, 'publishing'), (0.018475446607878098, 'international'), (0.017364456288797143, 'provenance'), (0.01267773954212696, 'springer'), (0.011417707029543367, 'springer_international'), (0.010404296422559245, 'international_publishing'), (0.0078741670950046062, 'paper'), (0.0072400520729634238, 'pose'), (0.0067710473589843662, 'information')], -2.2814611022990876), ([(0.055986015868153306, 'model'), (0.025958457738937974, 'science'), (0.022583624171598298, 'temporal'), (0.022074555515663095, 'computer'), (0.021897666724491428, 'system'), (0.019027190196859119, 'theoretical'), (0.017244999834073058, 'logic'), (0.01623827628951029, 'theoretical_computer'), (0.015997738318661549, 'checking'), (0.014968098582312662, 'property')], -2.2808162081922108), ([(0.083497780259651766, 'query'), (0.021734812935451767, 'data'), (0.020809299582586812, 'database'), (0.018522656592212819, 'view'), (0.013154852734081022, 'answering'), (0.011964624390680819, 'problem'), (0.010902784401680044, 'dependency'), (0.01046881022976648, 'answer'), (0.010026502003193175, 'complexity'), (0.0097142183938149565, 'rule')], -2.2789547407881772), ([(0.057402129036671801, 'ontology'), (0.018167628447941266, 'workshop'), (0.01430635196591709, 'semantic'), (0.01365930438681124, 'ceur'), (0.013389955071430244, 'proceeding'), (0.010406355227251966, 'semantic_web'), (0.008983595677260715, 'language'), (0.007685659027909535, 'w'), (0.0075803085765897005, 'ceur_workshop'), (0.0066739220359750924, 'knowledge')], -2.2566322525159359), ([(0.032841619606335366, 'configuration'), (0.026748093616057921, 'instruction'), (0.02449414093433425, 'simulation'), (0.016114579260143393, 'simulator'), (0.014378309510854117, 'space'), (0.011183187180856561, 'set'), (0.0099020196828146657, 'path'), (0.0095977546998839147, 'embedded'), (0.0090166888614326562, 'instruction_set'), (0.0085977833889685964, 'system')], -2.2484466788063737), ([(0.033564249828159143, 'motion'), (0.017605560873748306, 'data'), (0.014897514403376973, 'method'), (0.011010685424378629, 'using'), (0.010841777493833572, 'based'), (0.0093284948388263986, 'human'), (0.009011436601878673, 'character'), (0.0084926456257592153, 'system'), (0.0079817288717093215, 'body'), (0.0074847357944795006, '3d')], -2.2426609107044611), ([(0.029404715603898436, 'learning'), (0.020074731824905403, 'social'), (0.016706954779397737, 'machine'), (0.015639802581856526, 'research'), (0.010460020111175656, 'tool'), (0.010003939439890317, 'system'), (0.009671437776893485, 'machine_learning'), (0.009270773712180283, 'student'), (0.0085530330428470504, 'international'), (0.0084929022366421105, 'development')], -2.2276524817035588), ([(0.038651138966224942, 'language'), (0.032583923431147369, 'type'), (0.02021994466468709, 'programming'), (0.012809373851473807, 'semantics'), (0.012481853632662283, 'system'), (0.010356379639237542, 'specification'), (0.010128607940539408, 'logic'), (0.0091981808137533878, 'order'), (0.008854554109123336, 'functional'), (0.007684204758320227, 'programming_language')], -2.1754768711088834), ([(0.03416769575048155, 'system'), (0.028030538289550246, 'language'), (0.015583003758474092, 'text'), (0.014597260803094274, 'corpus'), (0.011822775326432985, 'natural'), (0.011593943246670249, 'generation'), (0.011149783926875717, 'natural_language'), (0.010446957626916453, 'paper'), (0.010278050688909524, 'annotation'), (0.008322739799724246, 'resource')], -2.1246174722543332), ([(0.040978139777340043, 'model'), (0.020756908988433152, 'neural'), (0.01443032791315408, 'data'), (0.013911072560360201, 'learning'), (0.012616584969884487, 'network'), (0.008858075822106173, 'parameter'), (0.0082448677441243449, 'bayesian'), (0.0073919590990998794, 'gaussian'), (0.0073294722544588247, 'using'), (0.0070920435170931297, 'neural_network')], -2.010235871824301), ([(0.015037269683342009, 'application'), (0.012289688665365561, 'system'), (0.010053528401889117, 'problem'), (0.0099354375006634899, 'cloud'), (0.0088286437847262186, 'resource'), (0.0075505160398986991, 'network'), (0.007015382034580569, 'paper'), (0.0063681435733103824, 'requirement'), (0.0063647340607685425, 'constraint'), (0.0055268467740196194, 'based')], -2.0090799777248174), ([(0.026312624180441845, 'process'), (0.025922271865681391, 'model'), (0.02536219660039421, 'system'), (0.024328188185760411, 'graph'), (0.018785152172478996, 'stochastic'), (0.01666739895263844, 'algebra'), (0.015882055323351168, 'modelling'), (0.01469959035591939, 'pepa'), (0.011903181827769296, 'analysis'), (0.010719754667156283, 'performance')], -2.0090104411796812), ([(0.036029208721865068, 'program'), (0.028532616795443624, 'proof'), (0.017926119355550576, 'theorem'), (0.015661657235946681, 'logic'), (0.0095015304870083913, 'technique'), (0.0083619666923850592, 'system'), (0.008145186241481547, 'proving'), (0.0080988570024568395, 'formal'), (0.0069546417438215201, 'verification'), (0.0067014237175301522, 'programming')], -1.9689922325492522), ([(0.037597341908906595, 'problem'), (0.031076971950883974, 'algorithm'), (0.017829488402090529, 'time'), (0.013139146597297428, 'polynomial'), (0.012080285125180919, 'bound'), (0.011954480720256351, 'show'), (0.011764801361378316, 'complexity'), (0.010386741104603522, 'state'), (0.008897357924355068, 'given'), (0.0087477733676816702, 'optimal')], -1.9617313847700406), ([(0.017659197320123755, 'gene'), (0.014887540387163174, 'protein'), (0.014460026658303057, 'biology'), (0.013479140953644001, 'expression'), (0.012018446602707846, 'analysis'), (0.011334550473862684, 'pathway'), (0.0099382391506894485, 'biological'), (0.0082228114681055543, 'system'), (0.0076737108510052275, 'system_biology'), (0.0070253832466884826, 'transcription')], -1.9499007249037155), ([(0.024437051733858007, 'system'), (0.020544311827805251, 'web'), (0.016783787425694939, 'service'), (0.0093743280251665576, 'application'), (0.0091800119399082971, 'technology'), (0.0084364539450134252, 'project'), (0.008319494641623135, 'user'), (0.0077918251351196911, 'support'), (0.0077591913876388519, 'distributed'), (0.0077261974473033287, 'paper')], -1.8972858140086082), ([(0.02126507452851701, 'parallel'), (0.018330168883412379, 'performance'), (0.015841546725273398, 'system'), (0.015542812551362303, 'application'), (0.015125530372723083, 'memory'), (0.012093564205049, 'program'), (0.011098826165686561, 'data'), (0.0097003470503829233, 'processor'), (0.0096517018470121795, 'approach'), (0.0095497197345011473, 'architecture')], -1.8010189550635261), ([(0.022143494135859639, 'reasoning'), (0.016320575244728137, 'planning'), (0.0155866605650972, 'proof'), (0.014694531706450921, 'knowledge'), (0.013978120767377457, 'system'), (0.012949665195419522, 'automated'), (0.009551355795233309, 'proceeding'), (0.0095417954758289791, 'plan'), (0.0092101895837004497, 'conference'), (0.0091219099239548181, 'artificial')], -1.756066939537551), ([(0.024422816517415138, 'cell'), (0.016094310076466119, 'neuron'), (0.014666924726849878, 'synaptic'), (0.011532435871799586, 'network'), (0.010803704894801794, 'activity'), (0.010741430702549374, 'model'), (0.0095643623027639492, 'dynamic'), (0.0089614544329956972, 'response'), (0.008317938875947891, 'mechanism'), (0.0077520763070025494, 'effect')], -1.7536766024428847), ([(0.020668809299140285, 'model'), (0.019228183666045937, 'inference'), (0.015463949595885408, 'method'), (0.015009029275975017, 'approach'), (0.013600670475372256, 'dynamic'), (0.010132252826304727, 'system'), (0.0097274242812547072, 'test'), (0.0084561185760654942, 'network'), (0.0083201865634401889, 'based'), (0.0076767646465321689, 'structure')], -1.74385931412425), ([(0.036340230773503207, 'image'), (0.028273046929426785, 'object'), (0.022086408415286266, 'model'), (0.016050635022837725, 'learning'), (0.011752703236849722, 'vision'), (0.011329826656478574, 'method'), (0.010979229398147774, 'class'), (0.0092102988963438451, 'visual'), (0.0090038995361805211, 'recognition'), (0.0084087000179618732, 'approach')], -1.728718272600819), ([(0.022392443359091699, 'recognition'), (0.021897717304748415, 'speech'), (0.018551381669471356, 'network'), (0.016112140386651739, 'using'), (0.015331058801967053, 'acoustic'), (0.01473199149643266, 'ieee'), (0.012937312118807253, 'model'), (0.011974294874719411, 'feature'), (0.011749959354003419, 'deep'), (0.011386376495791399, 'neural')], -1.7059419975369403), ([(0.032776230165730488, 'agent'), (0.025126086103076914, 'robot'), (0.02077551534224957, 'control'), (0.016682431885608379, 'system'), (0.011527765385159369, 'learning'), (0.011278968516807028, 'dynamic'), (0.010186357460057624, 'task'), (0.0096184403588140555, 'action'), (0.0088233855883370748, 'interaction'), (0.0087942997929892283, 'approach')], -1.6813409351598365), ([(0.013890924280662333, 'ieee'), (0.013557829014015541, 'performance'), (0.0098501347654588473, 'energy'), (0.0096038201436372105, 'cache'), (0.0094776829269342076, 'computer'), (0.0069376409205399526, 'international'), (0.0069193492994726368, 'technique'), (0.0067509575791530304, 'conference'), (0.0065402521617216281, 'high'), (0.0063091067140566576, 'architecture')], -1.6464321525653465), ([(0.023544819789572288, 'computational'), (0.022585670066725159, 'linguistics'), (0.020481724987521304, 'association'), (0.020142361278363937, 'language'), (0.019890749269991282, 'model'), (0.014505346289489737, 'association_computational'), (0.011809516194208022, 'proceeding'), (0.010977253078824165, 'grammar'), (0.0094636049027649726, 'semantic'), (0.0091262008354437915, 'word')], -1.5850656016800242), ([(0.073363358048375529, 'springer'), (0.058034621504289376, 'berlin'), (0.0558302063195125, 'heidelberg'), (0.032441251296406654, 'berlin_heidelberg'), (0.024894053779156395, 'international'), (0.023404518888709158, 'springer_berlin'), (0.019618264114578021, 'proceeding'), (0.019169479088295052, 'conference'), (0.016616469755979902, 'heidelberg_springer'), (0.011838403017986483, 'verlag')], -1.442718697333901), ([(0.069226349228813808, 'speech'), (0.021347335048102357, 'synthesis'), (0.015677521555276706, 'speaker'), (0.014684012742502563, 'voice'), (0.012222830965961794, 'system'), (0.011174047251217255, 'based'), (0.010878449869468124, 'speech_synthesis'), (0.0098927031529280982, 'model'), (0.0096197873362116552, 'synthetic'), (0.0093255130537298232, 'using')], -1.2841948830184344), ([(0.071096148385971045, 'translation'), (0.035988157240178752, 'machine'), (0.022938136053531274, 'machine_translation'), (0.022378831515617797, 'system'), (0.015842845106137274, 'statistical'), (0.014214401464520045, 'language'), (0.011820094195629544, 'model'), (0.011190674036002185, 'computational'), (0.010706671931361657, 'association'), (0.010619604963958492, 'based')], -1.1292860565951059), ([(0.082565924364109741, 'acm'), (0.039047342809420911, 'york'), (0.038367172644181442, 'new'), (0.03808772516784651, 'new_york'), (0.03651326046428989, 'usa'), (0.031912380102597303, 'ny'), (0.03078590135471582, 'ny_usa'), (0.0264179935144589, 'proceeding'), (0.018389106861186586, 'database'), (0.015389331457323326, 'acm_acm')], -1.0718514731842694)]
In [34]:
cm = CoherenceModel(
model=lda, texts=docs, dictionary=dictionary, coherence='c_v')
In [35]:
print((cm.get_coherence()))
2018-01-20 01:21:05,323 : INFO : using ParallelWordOccurrenceAccumulator(processes=3, batch_size=64) to estimate probabilities from sliding windows 2018-01-20 01:21:31,600 : INFO : serializing accumulator to return to master... 2018-01-20 01:21:31,733 : INFO : serializing accumulator to return to master... 2018-01-20 01:21:31,956 : INFO : serializing accumulator to return to master... 2018-01-20 01:21:31,964 : INFO : accumulator serialized 2018-01-20 01:21:31,606 : INFO : accumulator serialized 2018-01-20 01:21:31,742 : INFO : accumulator serialized 2018-01-20 01:21:32,477 : INFO : 3 accumulators retrieved from output queue 2018-01-20 01:21:33,206 : INFO : accumulated word occurrence stats for 95966 virtual documents
0.439834158888
Visualisation¶
In [43]:
vis_data = gensimvis.prepare(lda, corpus, dictionary, mds='tsne')
/Users/weiting/miniconda3/envs/infnet3/lib/python3.6/site-packages/pyLDAvis/_prepare.py:387: DeprecationWarning: .ix is deprecated. Please use .loc for label based indexing or .iloc for positional indexing See the documentation here: http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated topic_term_dists = topic_term_dists.ix[topic_order]
In [44]:
pyLDAvis.display(vis_data)
Out[44]:
In [22]:
# save in html for viewing
pyLDAvis.save_html(vis_data,open('ldavis.fullpub_pdf2txt.html','w'))
In [23]:
HTML("./ldavis.fullpub.html")
Out[23]:
Model 2: 2012-2017 LDA¶
Instead of using the entire collection, we use the collection from the past five years
In [24]:
pub_toks.drop('bow', axis=1, inplace=True)
gb = pub_toks.groupby('year')
In [25]:
sixYears = ['2017', '2016', '2015', '2014', '2013', '2012']
pd_years = {}
print("Year:\tPub count")
for yr, group in gb:
if yr in sixYears:
pd_years[yr] = group
print(("{}:\t{}".format(yr, len(group))))
Year: Pub count 2012: 573 2013: 623 2014: 629 2015: 583 2016: 645 2017: 521
In [26]:
combined_yrs = pd.concat(list(pd_years.values()))
In [27]:
print(("Total publications: ",len(combined_yrs)))
('Total publications: ', 3574)
In [28]:
combined_yrs.head(3)
Out[28]:
| year | title | abstract | publications | summary | summary_toks | |
|---|---|---|---|---|---|---|
| pub_id | ||||||
| a6c66577-d952-4d24-baa3-2707102e7470 | 2015 | probabilistic programs as spreadsheet queries | we describe the design, semantics, and impleme... | programming languages and systems,24th europea... | probabilistic programs as spreadsheet queries ... | [probabilist, program, spreadsheet, queri, des... |
| 7f8c35ed-b32c-4fc2-9a90-dc63961a12cb | 2015 | artificial personality and disfluency | the focus of this paper is artificial voices w... | interspeech 2015 16th annual conference of the... | artificial personality and disfluency the focu... | [artifici, person, disfluenc, focus, paper, ar... |
| 04cc8d28-b54c-40e9-9ec3-6746a37319f8 | 2015 | anchoring knowledge in interaction towards a h... | we outline a proposal for a research program l... | artificial general intelligence,8th internatio... | anchoring knowledge in interaction towards a h... | [anchor, knowledg, interact, harmon, subsymbol... |
In [29]:
toks_6yr = combined_yrs.summary_toks.tolist() # List of List
In [30]:
# Create the dictionary mapping:
dictionary_6yr = Dictionary(toks_6yr)
dictionary_6yr.filter_extremes(no_below=20, no_above=0.1)
dictionary_6yr.compactify()
# Create a bow tagging for each publication:
bowified = lambda row: dictionary_6yr.doc2bow(row.summary_toks)
combined_yrs['bow'] = combined_yrs.apply(bowified, axis=1)
In [31]:
corpus_6yr = combined_yrs.bow.tolist()
In [32]:
# Save the dictionary:
dictionary_6yr.save('../data/6yr/dictionary_6yr.pkl', pickle_protocol=-1)
# save the pub_toks:
pkl.dump(combined_yrs, open('../data/6yr/pub_toks_wBOW_6yr.pkl','wb'), -1)
# save the corpus:
pkl.dump(corpus_6yr, open('../data/6yr/corpus_6yr.pkl', 'wb'), -1)
LDA Modelling¶
In [ ]:
lda_6yr = models.ldamodel.LdaModel(
corpus=corpus_6yr,
id2word=dictionary_6yr,
num_topics=num_topics,
iterations=100,
passes=20,
random_state=rng)
In [34]:
# Save the model:
lda_6yr.save('../data/6yr/6yr.ldamodel')
Evaluate the model¶
In [38]:
cm = CoherenceModel(
model=lda_6yr, texts=toks_6yr, dictionary=dictionary_6yr, coherence='c_v')
In [39]:
print((cm.get_coherence()))
0.471073918311
Visualisation¶
In [35]:
vis_data_6yr = gensimvis.prepare(lda_6yr, corpus_6yr, dictionary_6yr, mds='tsne')
/Users/weiting/miniconda3/envs/infnet/lib/python2.7/site-packages/pyLDAvis/_prepare.py:387: DeprecationWarning: .ix is deprecated. Please use .loc for label based indexing or .iloc for positional indexing See the documentation here: http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated topic_term_dists = topic_term_dists.ix[topic_order]
In [36]:
pyLDAvis.display(vis_data_6yr)
Out[36]:
In [37]:
# save in html for viewing
pyLDAvis.save_html(vis_data_6yr, open('ldavis.6yr.html','w'))
In [3]:
HTML("./ldavis.6yr.html")
Out[3]:
Model 2 with pdf2txt¶
In [25]:
pub_pdf2txt.drop('bow', axis=1, inplace=True)
gb = pub_pdf2txt.groupby('year')
In [26]:
sixYears = ['2017', '2016', '2015', '2014', '2013', '2012']
pd_years = {}
print("Year:\tPub count")
for yr, group in gb:
if yr in sixYears:
pd_years[yr] = group
print(("{}:\t{}".format(yr, len(group))))
Year: Pub count 2012: 573 2013: 623 2014: 629 2015: 583 2016: 645 2017: 521
In [27]:
combined_yrs = pd.concat(list(pd_years.values()))
In [28]:
print(("Total publications: ",len(combined_yrs)))
('Total publications: ', 3574)
In [29]:
combined_yrs.head(3)
Out[29]:
| year | title | abstract | publications | pdf2txt | combined | toks | |
|---|---|---|---|---|---|---|---|
| pub_id | |||||||
| a6c66577-d952-4d24-baa3-2707102e7470 | 2015 | probabilistic programs as spreadsheet queries | we describe the design, semantics, and impleme... | programming languages and systems,24th europea... | Probabilistic Programs as Spreadsheet Queries\... | probabilistic programs as spreadsheet queries.... | [probabilist, program, spreadsheet, queri, des... |
| 7f8c35ed-b32c-4fc2-9a90-dc63961a12cb | 2015 | artificial personality and disfluency | the focus of this paper is artificial voices w... | interspeech 2015 16th annual conference of the... | Artificial Personality and Disfluency\n \n Cit... | artificial personality and disfluency. the foc... | [artifici, person, disfluenc, focus, paper, ar... |
| 04cc8d28-b54c-40e9-9ec3-6746a37319f8 | 2015 | anchoring knowledge in interaction towards a h... | we outline a proposal for a research program l... | artificial general intelligence,8th internatio... | Anchoring Knowledge in Interaction: Towards a ... | anchoring knowledge in interaction towards a h... | [anchor, knowledg, interact, harmon, subsymbol... |
In [31]:
toks_6yr_pdf2txt = combined_yrs.toks.tolist() # List of List
In [33]:
# Create the dictionary mapping:
dictionary_6yr_pdf2txt = Dictionary(toks_6yr_pdf2txt)
dictionary_6yr_pdf2txt.filter_extremes(no_below=20, no_above=0.1)
dictionary_6yr_pdf2txt.compactify()
# Create a bow tagging for each publication:
bowified = lambda row: dictionary_6yr_pdf2txt.doc2bow(row.toks)
combined_yrs['bow'] = combined_yrs.apply(bowified, axis=1)
In [34]:
corpus_6yr_pdf2txt = combined_yrs.bow.tolist()
In [35]:
# Save the dictionary:
dictionary_6yr_pdf2txt.save('../data/6yr_pdf2txt/dictionary_6yr_pdf2txt.pkl', pickle_protocol=-1)
# save the pub_toks:
pkl.dump(combined_yrs, open('../data/6yr_pdf2txt/pub_toks_wBOW_6yr_pdf2txt.pkl','wb'), -1)
# save the corpus:
pkl.dump(corpus_6yr_pdf2txt, open('../data/6yr_pdf2txt/corpus_6yr_pdf2txt.pkl', 'wb'), -1)
LDA Modelling¶
In [ ]:
lda_6yr_pdf2txt = models.ldamodel.LdaModel(
corpus=corpus_6yr_pdf2txt,
id2word=dictionary_6yr_pdf2txt,
num_topics=num_topics,
iterations=100,
passes=20,
random_state=rng)
In [ ]:
# Save the model:
lda_6yr_pdf2txt.save('../data/6yr_pdf2txt/6yr.ldamodel')
Evaluate the model¶
In [ ]:
cm = CoherenceModel(
model=lda_6yr_pdf2txt, texts=toks_6yr_pdf2txt, dictionary=dictionary_6yr_pdf2txt, coherence='c_v')
In [ ]:
print((cm.get_coherence()))
Visualisation¶
In [ ]:
vis_data_6yr_pdf2txt = gensimvis.prepare(lda_6yr_pdf2txt, corpus_6yr_pdf2txt, dictionary_6yr_pdf2txt, mds='tsne')
In [ ]:
pyLDAvis.display(vis_data_6yr_pdf2txt)
In [ ]:
# save in html for viewing
pyLDAvis.save_html(vis_data_6yr_pdf2txt, open('ldavis.6yr_pdf2txt.html','w'))
In [ ]:
HTML("./ldavis.6yr_pdf2txt.html")
Misc¶
In [ ]:
length = combined_yrs.apply(lambda row: len(row.summary_toks),axis=1)
In [ ]:
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(111)
plt.hist(length,bins=100)
plt.show()
In [ ]:
# we can check the distribution:
dictionary_6yr.doc2bow(['tree'])
In [ ]:
lda_6yr.get_topic_terms(0)